fs/posix: On windows use helpers and init format.json properly. (#3434)

Fixes #3433

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,33 @@
+<!--- Provide a general summary of the issue in the Title above -->
+
+## Expected Behavior
+<!--- If you're describing a bug, tell us what should happen -->
+<!--- If you're suggesting a change/improvement, tell us how it should work -->
+
+## Current Behavior
+<!--- If describing a bug, tell us what happens instead of the expected behavior -->
+<!--- If suggesting a change/improvement, explain the difference from current behavior -->
+
+## Possible Solution
+<!--- Not obligatory, but suggest a fix/reason for the bug, -->
+<!--- or ideas how to implement the addition or change -->
+
+## Steps to Reproduce (for bugs)
+<!--- Provide a link to a live example, or an unambiguous set of steps to -->
+<!--- reproduce this bug. Include code to reproduce, if relevant -->
+1.
+2.
+3.
+4.
+
+## Context
+<!--- How has this issue affected you? What are you trying to accomplish? -->
+<!--- Providing context helps us come up with a solution that is most useful in the real world -->
+
+## Your Environment
+<!--- Include as many relevant details about the environment you experienced the bug in -->
+* Version used:
+* Environment name and version (e.g. nginx 1.9.1):
+* Server type and version:
+* Operating System and version:
+* Link to your project:

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,28 @@
+<!--- Provide a general summary of your changes in the Title above -->
+
+## Description
+<!--- Describe your changes in detail -->
+
+## Motivation and Context
+<!--- Why is this change required? What problem does it solve? -->
+<!--- If it fixes an open issue, please link to the issue here. -->
+
+## How Has This Been Tested?
+<!--- Please describe in detail how you tested your changes. -->
+<!--- Include details of your testing environment, and the tests you ran to -->
+<!--- see how your change affects other areas of the code, etc. -->
+
+## Types of changes
+<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to change)
+
+## Checklist:
+<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
+<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
+- [ ] My code follows the code style of this project.
+- [ ] My change requires a change to the documentation.
+- [ ] I have updated the documentation accordingly.
+- [ ] I have added tests to cover my changes.
+- [ ] All new and existing tests passed.

.gitignore

@@ -9,5 +9,10 @@ site/
 /.idea/
 /Minio.iml
 **/access.log
-ui-assets.go
-ui-assets.asc
+build
+vendor/**/*.js
+vendor/**/*.json
+release
+.DS_Store
+*.syso
+coverage.txt

.mention-bot

@@ -0,0 +1,4 @@
+{
+    "numFilesToCheck": 10,
+    "requiredOrgs": ["minio"]
+}

.travis.yml

@@ -1,24 +1,24 @@
-sudo: false
+go_import_path: github.com/minio/minio
+sudo: required
+
+dist: trusty
+
 language: go
 
 os:
 - linux
-- osx
-
-osx_image: xcode7.2
 
 env:
 - ARCH=x86_64
 - ARCH=i686
 
 script:
-- make test
+- make
 - make test GOFLAGS="-race"
+- make coverage
+
+after_success:
+- bash <(curl -s https://codecov.io/bash)
 
 go:
-- 1.5.2
-- 1.5.3
-
-notifications:
-  slack:
-    secure: K9tsn5MvrCAxuEZTxn+m3Kq1K2NG2xMEJFSv/sTp+RQBW7TslPHzv859GsIvrm8mU1y1btOU9RlOzqrRUczI5cJpE8IL1oljPZbXrIXgetE0kbsw0Wpy99g27UQ2VGp933WDu8tfj7zU4cZv+BI0RltNLwqYO6GWXmcWP0IueCU=
+- 1.7.3

AWS-SDK-GO.md

@@ -1,76 +0,0 @@
-## Using aws-sdk-go with Minio
-
-aws-sdk-go is the official AWS SDK for the Go programming language. This document covers
-how to use aws-sdk-go with Minio server.
-
-### Install AWS SDK S3 service
-
-```sh
-$ go get github.com/aws/aws-sdk-go/service/s3
-```
-
-### List all buckets on Minio
-
-```go
-package main
-
-import (
-	"fmt"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-)
-
-func main() {
-	newSession := session.New()
-	s3Config := &aws.Config{
-		Credentials: credentials.NewStaticCredentials("<YOUR-ACCESS-KEY-ID>", "<YOUR-SECRET-ACCESS-KEY", ""),
-		Endpoint:         aws.String("http://localhost:9000"),
-		Region:           aws.String("us-east-1"),
-		DisableSSL:       aws.Bool(true),
-		S3ForcePathStyle: aws.Bool(true),
-	}
-	// Create an S3 service object in the default region.
-	s3Client := s3.New(newSession, s3Config)
-
-	cparams := &s3.CreateBucketInput{
-		Bucket: aws.String("newbucket"), // Required
-	}
-	_, err := s3Client.CreateBucket(cparams)
-	if err != nil {
-		// Message from an error.
-		fmt.Println(err.Error())
-		return
-	}
-
-	var lparams *s3.ListBucketsInput
-	// Call the ListBuckets() Operation
-	resp, err := s3Client.ListBuckets(lparams)
-	if err != nil {
-		// Message from an error.
-		fmt.Println(err.Error())
-		return
-	}
-
-	// Pretty-print the response data.
-	fmt.Println(resp)
-}
-```
-
-Populate your AccessKeyId and SecretAccessKey credentials and run the program as shown below.
-
-```sh
-$ go run aws-sdk-minio.go
-{
-  Buckets: [{
-      CreationDate: 2015-10-22 01:46:04 +0000 UTC,
-      Name: "newbucket"
-    }],
-  Owner: {
-    DisplayName: "minio",
-    ID: "minio"
-  }
-}
-```

Browser.md

@@ -1,39 +0,0 @@
-## Minio Browser
-
-Minio Browser uses Json Web Tokens to authenticate JSON RPC requests.
-
-Initial request generates a token for 'AccessKey' and 'SecretKey'
-provided by the user.
-
-<blockquote>
-Currently these tokens expire after 10hrs, this is not configurable yet.
-</blockquote>
-
-### Start minio server
-
-```
-minio server <testdir>
-```
-
-### JSON RPC APIs.
-
-JSON RPC namespace is `Web`.
-
-#### Auth Operations
-
-* Login - waits for 'username, password' and on success replies a new Json Web Token (JWT).
-* ResetToken - resets token, requires password and token.
-* Logout - currently a dummy operation.
-
-#### Bucket/Object Operations.
-
-* ListBuckets - lists buckets, requires a valid token.
-* ListObjects - lists objects, requires a valid token.
-* MakeBucket - make a new bucket, requires a valid token.
-* GetObjectURL - generates a URL for download access, requires a valid token.
-  (generated URL is valid for 1hr)
-* PutObjectURL - generates a URL for upload access, requies a valid token.
-  (generated URL is valid for 1hr)
-
-#### Server Operations. 
-* DiskInfo - get backend disk statistics.

CONTRIBUTING.md

@@ -1,13 +1,13 @@
 ### Install Golang
 
-If you do not have a working Golang environment setup please follow [Golang Installation Guide](./INSTALLGO.md).
+If you do not have a working Golang environment setup please follow [Golang Installation Guide](https://docs.minio.io/docs/how-to-install-golang).
 
 ### Setup your Minio Github Repository
-Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL and pass it to ``go get`` command. Go uses git to clone a copy into your project workspace folder.
+Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL for minio from your personal github repo (you will need it for the `git clone` command below).
 ```sh
 $ mkdir -p $GOPATH/src/github.com/minio
 $ cd $GOPATH/src/github.com/minio
-$ git clone https://github.com/$USER_ID/minio
+$ git clone <paste saved URL for personal forked minio repo>
 $ cd minio
 ```
 
@@ -51,7 +51,7 @@ Building Libraries
     - Push to the branch (git push origin my-new-feature)
     - Create new Pull Request
 
-* If you have additional dependencies for ``Minio``, ``Minio`` manages its depedencies using [govendor](https://github.com/kardianos/govendor)
+* If you have additional dependencies for ``Minio``, ``Minio`` manages its dependencies using [govendor](https://github.com/kardianos/govendor)
     - Run `go get foo/bar`
     - Edit your code to import foo/bar
     - Run `make pkg-add PKG=foo/bar` from top-level directory

Docker.md

@@ -1,32 +0,0 @@
-## Running Minio in Docker.
-
-### Installing Docker.
-
-```bash
-sudo apt-get install Docker.io
-```
-
-### Generating `minio configs` for the first time.
-
-```bash
-docker run -p 9000:9001 minio/minio:latest
-```
-
-### Persist `minio configs`.
-
-```bash
-docker commit <running_minio_container_id> minio/my-minio
-docker stop <running_minio_container_id>
-```
-
-### Create a data volume container.
-
-```bash
-docker create -v /export --name minio-export minio/my-minio /bin/true
-```
-
-You can then use the `--volumes-from` flag to mount the `/export` volume in another container.
-
-```bash
-docker run -p 9000:9001 --volumes-from minio-export --name minio1 minio/my-minio
-```

Dockerfile

@@ -1,9 +1,18 @@
-# use "make dockerimage" to build
-FROM scratch
-ENV DOCKERIMAGE 1
-ADD minio.dockerimage /minio
-ADD export /export
+FROM golang:1.7-alpine
+
+WORKDIR /go/src/app
+
+COPY . /go/src/app
+
+RUN \
+	apk add --no-cache git && \
+	go-wrapper download && \
+	go-wrapper install -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
+	mkdir -p /export/docker && \
+	cp /go/src/app/docs/docker/README.md /export/docker/ && \
+	rm -rf /go/pkg /go/src && \
+	apk del git
+
 EXPOSE 9000
-EXPOSE 9001
-ENTRYPOINT ["/minio"]
-CMD ["server", "/export"]
+ENTRYPOINT ["minio"]
+VOLUME ["/export"]

INSTALLGO.md

@@ -1,84 +0,0 @@
-## Ubuntu (Kylin) 14.04
-### Build Dependencies
-This installation document assumes Ubuntu 14.04+ on x86-64 platform.
-
-##### Install Git, GCC
-```sh
-$ sudo apt-get install git build-essential
-```
-
-##### Install Go 1.5+
-
-Download Go 1.5+ from [https://golang.org/dl/](https://golang.org/dl/).
-
-```sh
-$ wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
-$ mkdir -p ${HOME}/bin/
-$ mkdir -p ${HOME}/go/
-$ tar -C ${HOME}/bin/ -xzf go1.5.1.linux-amd64.tar.gz
-```
-##### Setup GOROOT and GOPATH
-
-Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
-and GOPATH specifies the location of your project workspace.
-
-```sh
-export GOROOT=${HOME}/bin/go
-export GOPATH=${HOME}/go
-export PATH=$PATH:${HOME}/bin/go/bin:${GOPATH}/bin
-```
-##### Source the new enviornment
-
-```sh
-$ source ~/.bashrc
-```
-
-##### Testing it all
-
-```sh
-$ go env
-```
-
-## OS X (Yosemite) 10.10
-### Build Dependencies
-This installation document assumes OS X Yosemite 10.10+ on x86-64 platform.
-
-##### Install brew
-```sh
-$ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-```
-
-##### Install Git, Python
-```sh
-$ brew install git python
-```
-
-##### Install Go 1.5+
-
-Install golang binaries using `brew`
-
-```sh
-$ brew install go
-$ mkdir -p $HOME/go
-```
-
-##### Setup GOROOT and GOPATH
-
-Add the following exports to your ``~/.bash_profile``. Environment variable GOROOT specifies the location of your golang binaries
-and GOPATH specifies the location of your project workspace.
-
-```sh
-export GOPATH=${HOME}/go
-export GOVERSION=$(brew list go | head -n 1 | cut -d '/' -f 6)
-export GOROOT=$(brew --prefix)/Cellar/go/${GOVERSION}/libexec
-export PATH=$PATH:${GOPATH}/bin
-```
-##### Source the new enviornment
-```sh
-$ source ~/.bash_profile
-```
-##### Testing it all
-
-```sh
-$ go env
-```

MAINTAINERS.md

@@ -18,7 +18,7 @@ $
 
   - Run `go get foo/bar`
   - Edit your code to import foo/bar
-  - Run `govendor add foo/bar` from top-level folder
+  - Run `govendor add foo/bar` from top-level directory
 
 #### Remove dependencies 
 
@@ -34,4 +34,4 @@ $
 
 `minio` doesn't follow semantic versioning style, `minio` instead uses the release date and time as the release versions.
 
-`make release` will install new released binary into your `GOPATH`
+`make release` will generate new binary into `release` directory.

Makefile

@@ -1,103 +1,145 @@
 LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
-DOCKER_BIN := $(shell which docker)
-DOCKER_LDFLAGS := '$(LDFLAGS) -extldflags "-static"'
+PWD := $(shell pwd)
+GOPATH := $(shell go env GOPATH)
 BUILD_LDFLAGS := '$(LDFLAGS)'
 TAG := latest
-UI_ASSETS := ui-assets.go
-UI_ASSETS_ARMOR := ui-assets.asc
+
+HOST ?= $(shell uname)
+CPU ?= $(shell uname -m)
+
+# if no host is identifed (no uname tool)
+# we assume a Linux-64bit build
+ifeq ($(HOST),)
+  HOST = Linux
+endif
+
+# identify CPU
+ifeq ($(CPU), x86_64)
+  HOST := $(HOST)64
+else
+ifeq ($(CPU), amd64)
+  HOST := $(HOST)64
+else
+ifeq ($(CPU), i686)
+  HOST := $(HOST)32
+endif
+endif
+endif
+
+
+#############################################
+# now we find out the target OS for
+# which we are going to compile in case
+# the caller didn't yet define OS himself
+ifndef (OS)
+  ifeq ($(HOST), Linux64)
+    arch = gcc
+  else
+  ifeq ($(HOST), Linux32)
+    arch = 32
+  else
+  ifeq ($(HOST), Darwin64)
+    arch = clang
+  else
+  ifeq ($(HOST), Darwin32)
+    arch = clang
+  else
+  ifeq ($(HOST), FreeBSD64)
+    arch = gcc
+  endif
+  endif
+  endif
+  endif
+  endif
+endif
 
 all: install
 
-checkdeps:
+checks:
 	@echo "Checking deps:"
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
+	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
-checkgopath:
-	@echo "Checking if project is at ${GOPATH}"
-	@for miniopath in $(echo ${GOPATH} | sed 's/:/\n/g'); do if [ ! -d ${miniopath}/src/github.com/minio/minio ]; then echo "Project not found in ${miniopath}, please follow instructions provided at https://github.com/minio/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository" && exit 1; fi done
-	@echo "Setting BUILD_LDFLAGS: ${BUILD_LDFLAGS}"
-
-checkdocker:
-	@echo "Checking if docker is installed.. "
-	@if [ -z ${DOCKER_BIN} ]; then echo "Docker not installed, cannot build docker image. Please install 'sudo apt-get install docker.io'" && exit 1; else echo "Docker installed at ${DOCKER_BIN}."; fi;
-
-getdeps: checkdeps checkgopath
+getdeps: checks
 	@go get -u github.com/golang/lint/golint && echo "Installed golint:"
-	@go get -u golang.org/x/tools/cmd/vet && echo "Installed vet:"
 	@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
 	@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:"
+	@go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell:"
+	@go get -u github.com/gordonklaus/ineffassign && echo "Installed ineffassign:"
 
-$(UI_ASSETS):
-	@curl -s https://dl.minio.io/assets/server/$(UI_ASSETS_ARMOR) 2>&1 > $(UI_ASSETS_ARMOR) && echo "Downloading signature file $(UI_ASSETS_ARMOR) for verification:"
-	@gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728 2>&1 > /dev/null && echo "Importing public key:"
-	@curl -s https://dl.minio.io/assets/server/$@ 2>&1 > $@ && echo "Downloading UI assets file $@:"
-	@gpg --batch --no-tty --verify $(UI_ASSETS_ARMOR) $@ 2>&1 > /dev/null && echo "Verifying signature of downloaded assets."
-
-verifiers: getdeps vet fmt lint cyclo
+verifiers: vet fmt lint cyclo spelling
 
 vet:
 	@echo "Running $@:"
-	@GO15VENDOREXPERIMENT=1 go tool vet -all *.go
+	@GO15VENDOREXPERIMENT=1 go tool vet -all ./cmd
 	@GO15VENDOREXPERIMENT=1 go tool vet -all ./pkg
-	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true *.go
+	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true ./cmd
 	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true ./pkg
 
 fmt:
 	@echo "Running $@:"
-	@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
+	@GO15VENDOREXPERIMENT=1 gofmt -s -l cmd
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
 
 lint:
 	@echo "Running $@:"
-	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
-	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint github.com/minio/minio/pkg...
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd...
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg...
+
+ineffassign:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/ineffassign .
 
 cyclo:
 	@echo "Running $@:"
-	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
-	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 100 cmd
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 100 pkg
 
 build: getdeps verifiers $(UI_ASSETS)
-	@echo "Installing minio:"
 
 deadcode:
-	@GO15VENDOREXPERIMENT=1 deadcode
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
+
+spelling:
+	@-GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell -error `find cmd/`
+	@-GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell -error `find pkg/`
+	@-GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell -error `find docs/`
 
 test: build
-	@echo "Running all testing:"
-	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
+	@echo "Running all minio testing:"
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/cmd...
 	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
 
+coverage: build
+	@echo "Running all coverage for minio:"
+	@GO15VENDOREXPERIMENT=1 ./buildscripts/go-coverage.sh
+
 gomake-all: build
+	@echo "Installing minio:"
 	@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
 
 pkg-add:
-	@GO15VENDOREXPERIMENT=1 govendor add $(PKG)
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
 
 pkg-update:
-	@GO15VENDOREXPERIMENT=1 govendor update $(PKG)
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor update $(PKG)
 
 pkg-remove:
-	@GO15VENDOREXPERIMENT=1 govendor remove $(PKG)
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor remove $(PKG)
+
+pkg-list:
+	@GO15VENDOREXPERIMENT=1 $(GOPATH)/bin/govendor list
 
 install: gomake-all
 
-dockerimage: checkdocker verifiers $(UI_ASSETS)
-	@echo "Building docker image:" minio:$(TAG)
-	@GO15VENDOREXPERIMENT=1 go build --ldflags $(DOCKER_LDFLAGS) -o minio.dockerimage
-	@mkdir -p export
-	@sudo docker build --rm --tag=minio/minio:$(TAG) .
-	@rmdir export
-	@rm minio.dockerimage
+release: verifiers
+	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
 
-release:
-	@./release.sh
+experimental: verifiers
+	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
 
 clean:
 	@echo "Cleaning up all the generated files:"
-	@rm -fv cover.out
-	@rm -fv minio
-	@rm -fv minio.test
-	@rm -fv pkg/fs/fs.test
-	@rm -fv ui-assets.go
-	@rm -fv ui-assets.asc
+	@find . -name '*.test' | xargs rm -fv
+	@rm -rf build
+	@rm -rf release

NOTICE

@@ -5,5 +5,5 @@ This product includes software developed at Minio, Inc.
 
 The Minio project contains unmodified/modified subcomponents too with
 separate copyright notices and license terms. Your use of the source
-code for the these subcomponents is subject to the terms and conditions
+code for these subcomponents is subject to the terms and conditions
 of the following licenses.

README.md

@@ -1,200 +1,99 @@
-## Minio [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+# Minio Quickstart Guide [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
 
-Minio is a distributed object storage server written in Golang. Source is available under free software / open source [Apache license 2.0](./LICENSE). API compatible with Amazon S3 cloud storage service.
+Minio is an object storage server released under Apache License v2.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
 
-## Description
+Minio server is light enough to be bundled with the application stack, similar to NodeJS, Redis and MySQL.
 
-Micro services environment provisions one Minio server per application instance. Scalability is achieved through large number of smaller personalized instances. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.
-
-## Minio Client
-
-[Minio Client (mc)](https://github.com/minio/mc#minio-client-mc-) provides a modern alternative to Unix commands like ``ls``, ``cat``, ``cp``, ``sync``, and ``diff``. It supports POSIX compatible filesystems and Amazon S3 compatible cloud storage systems. It is entirely written in Golang.
-
-## Amazon S3 Compatible Client Libraries
-- [Golang Library](https://github.com/minio/minio-go)
-- [Java Library](https://github.com/minio/minio-java)
-- [Nodejs Library](https://github.com/minio/minio-js)
-- [Python Library](https://github.com/minio/minio-py)
-- [.Net Library](https://github.com/minio/minio-dotnet)
-
-### Install [![Build Status](https://travis-ci.org/minio/minio.svg?branch=master)](https://travis-ci.org/minio/minio)[![Build status](https://ci.appveyor.com/api/projects/status/royh137dni8yevep/branch/master?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-qxbjq/branch/master)
-
-#### GNU/Linux
-
-Download ``minio`` for:
-
-- ``64-bit Intel`` from https://dl.minio.io/server/minio/release/linux-amd64/minio
-- ``32-bit Intel`` from https://dl.minio.io/server/minio/release/linux-386/minio
-- ``32-bit ARM`` from https://dl.minio.io/server/minio/release/linux-arm/minio
-
-~~~
-$ chmod +x minio
-$ ./minio --help
-~~~
-
-#### OS X
-
-Download ``minio`` from https://dl.minio.io/server/minio/release/darwin-amd64/minio
-
-~~~
-$ chmod 755 minio
-$ ./minio --help
-~~~
-
-#### Microsoft Windows
-
-Download ``minio`` for:
-
-- ``64-bit`` from https://dl.minio.io/server/minio/release/windows-amd64/minio.exe
-- ``32-bit`` from https://dl.minio.io/server/minio/release/windows-386/minio.exe
-
-~~~
-C:\Users\Username\Downloads> minio.exe --help
-~~~
-
-#### Docker container
-
-Download ``minio`` for docker.
-
-~~~
+## Docker Container
+### Stable
+```sh
 $ docker pull minio/minio
-~~~
-
-Read more here on [How to configure data volume containers for Minio?](./Docker.md)
-
-#### Source
-<blockquote>
-NOTE: Source installation is intended for only developers and advanced users.  For general use, please download official releases from https://minio.io/download.
-</blockquote>
-
-If you do not have a working Golang environment, please follow [Install Golang](./INSTALLGO.md).
-
-~~~
-$ go get -d github.com/minio/minio
-$ cd $GOPATH/src/github.com/minio/minio
-$ make
-~~~
-
-### How to use Minio?
-
-```
-NAME:
-  minio server - Start Minio cloud storage server.
-
-USAGE:
-  minio server [OPTION VALUE] PATH
-
-  OPTION = expiry        VALUE = NN[h|m|s] [DEFAULT=Unlimited]
-  OPTION = min-free-disk VALUE = NN% [DEFAULT: 10%]
-
-EXAMPLES:
-  1. Start minio server on Linux.
-        $ minio server /home/shared
-
-  2. Start minio server on Windows.
-        $ minio server C:\MyShare
-
-  3. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
-        $ minio --address 192.168.1.101:9000 server /home/shared
-
-  4. Start minio server with minimum free disk threshold to 5%
-        $ minio server min-free-disk 5% /home/shared/Pictures
-
-  5. Start minio server with minimum free disk threshold to 15% with auto expiration set to 1h
-        $ minio server min-free-disk 15% expiry 1h /home/shared/Documents
+$ docker run -p 9000:9000 minio/minio server /export
 ```
 
-#### Start Minio server.
+### Edge
+```sh
+$ docker pull minio/minio:edge
+$ docker run -p 9000:9000 minio/minio:edge server /export
+```
+Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
-~~~
+## OS X
+### Homebrew
+Install minio packages using [Homebrew](http://brew.sh/)
+
+```sh
+$ brew install minio
 $ minio server ~/Photos
-AccessKey: WLGDGYAQYIGI833EV05A  SecretKey: BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF  Region: us-east-1
-
-Minio Object Storage:
-    http://127.0.0.1:9000
-    http://10.0.0.3:9000
-
-Minio Browser:
-    http://127.0.0.1:9000
-    http://10.0.0.3:9000
-
-To configure Minio Client:
-    $ wget https://dl.minio.io/client/mc/release/darwin-amd64/mc
-    $ chmod 755 mc
-    $ ./mc config host add myminio http://localhost:9000 WLGDGYAQYIGI833EV05A BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF
-~~~
-
-#### How to use AWS CLI with Minio?
-
-<blockquote>
-This section assumes that you have already installed aws-cli, if not please visit https://aws.amazon.com/cli/
-</blockquote>
-
-To configure `aws-cli`, type `aws configure` and follow below steps.
-
-```
-$ aws configure
-AWS Access Key ID [None]: YOUR_ACCESS_KEY_HERE
-AWS Secret Access Key [None]: YOUR_SECRET_KEY_HERE
-Default region name [None]: us-east-1
-Default output format [None]: ENTER
 ```
 
-Additionally enable `aws-cli` to use AWS Signature Version '4' for Minio server.
-
-```
-$ aws configure set default.s3.signature_version s3v4
+### Binary Download
+| Platform| Architecture | URL|
+| ----------| -------- | ------|
+|Apple OS X|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio|
+```sh
+$ chmod 755 minio
+$ ./minio server ~/Photos
 ```
 
-To list your buckets.
-```
-$ aws --endpoint-url http://localhost:9000 s3 ls
-2016-01-07 16:38:23 testbucket
+## GNU/Linux
+### Binary Download
+| Platform| Architecture | URL|
+| ----------| -------- | ------|
+|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
+||32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio|
+||32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio|
+```sh
+$ chmod +x minio
+$ ./minio server ~/Photos
 ```
 
-To list contents inside bucket.
-```
-$ aws --endpoint-url http://localhost:9000 s3 ls s3://testbucket
-                           PRE test/
-2015-12-17 08:46:41   12232928 vim
-2016-01-07 16:38:23   32232928 emacs
-2015-12-09 08:05:24     138504 s3cmd
+## Microsoft Windows
+### Binary Download
+| Platform| Architecture | URL|
+| ----------| -------- | ------|
+|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
+||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
+```sh
+C:\Users\Username\Downloads> minio.exe server D:\Photos
 ```
 
-#### How to use AWS SDK with Minio?
+## FreeBSD
+### Binary Download
+| Platform| Architecture | URL|
+| ----------| -------- | ------|
+|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
+```sh
+$ chmod 755 minio
+$ ./minio server ~/Photos
+```
+Please visit official zfs FreeBSD guide for more details [here](https://www.freebsd.org/doc/handbook/zfs-quickstart.html)
 
-Please follow the documentation here - [Using aws-sdk-go with Minio server](./AWS-SDK-GO.md)
+## Install from Source
 
-#### How to use s3cmd with Minio?
+Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
 
-<blockquote>
-This section assumes that you have already installed s3cmd, if not please visit http://s3tools.org/s3cmd
-</blockquote>
+```sh
 
-Edit the following fields in your s3cmd configuration file `~/.s3cfg` .
+$ go get -u github.com/minio/minio
 
-```
-host_base = localhost:9000
-host_bucket = localhost:9000
-access_key = YOUR_ACCESS_KEY_HERE
-secret_key = YOUR_SECRET_KEY_HERE
 ```
 
-To list your buckets.
-```
-$ s3cmd ls s3://
-2015-12-09 16:12  s3://testbbucket
-```
+## Test using Minio Browser
+Minio Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully.
 
-To list contents inside buckets.
-```
-$ s3cmd ls s3://testbucket/
-                       DIR   s3://testbucket/test/
-2015-12-09 16:05    138504   s3://testbucket/newfile
-```
+![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
+
+## Test using Minio Client `mc`
+`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the Minio Client [Quickstart Guide](https://docs.minio.io/docs/minio-client-quickstart-guide) for further instructions.
+
+## Explore Further
+- [Minio Erasure Code QuickStart Guide](https://docs.minio.io/docs/minio-erasure-code-quickstart-guide)
+- [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
+- [Use `aws-cli` with Minio Server](https://docs.minio.io/docs/aws-cli-with-minio)
+- [Use `s3cmd` with Minio Server](https://docs.minio.io/docs/s3cmd-with-minio)
+- [Use `minio-go` SDK with Minio Server](https://docs.minio.io/docs/golang-client-quickstart-guide)
+- [The Minio documentation website](https://docs.minio.io)
 
 ## Contribute to Minio Project
-Please follow Minio [Contributor's Guide](./CONTRIBUTING.md)
-
-### Jobs
-If you think in Lisp or Haskell and hack in go, you would blend right in. Send your github link to callhome@minio.io.
+Please follow Minio [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)

README_ZH.md

@@ -0,0 +1,200 @@
+# Minio 快速入门 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+
+Minio是一个对象存储服务，基于Apache License v2.0协议. 它完全兼容亚马逊的S3云储存服务，非常适合于存储很多非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。 
+
+##  1. 下载
+
+Minio是一个非常轻量的服务,可以很简单的和其他应用的结合，类似 NodeJS, Redis 或者 MySQL.
+
+| Platform| Architecture | URL|
+| ----------| -------- | ------|
+|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
+||32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio|
+||32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio|
+|Apple OS X|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio|
+|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
+||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
+|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
+
+### Homebrew 安装
+
+使用[Homebrew](http://brew.sh/) 来安装minio
+ 
+```sh
+$ brew install minio
+$ minio --help
+```
+
+### 源码安装
+
+源码安装只针对开发者和一些高级用户，如果你还没有golang的环境，请安装golang官网安装[How to install Golang](https://docs.minio.io/docs/zh-CN/how-to-install-golang).
+
+```sh
+
+$ go get -u github.com/minio/minio
+
+
+```
+
+## 2. 运行Minio服务
+
+
+### GNU/Linux
+
+ ```sh
+
+$ chmod +x minio
+$ ./minio --help
+$ ./minio server ~/Photos
+
+端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
+AccessKey: USWUXHGYZQYFYFFIT3RE
+SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+区域:    us-east-1
+
+浏览器访问入口:
+   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
+
+命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
+   $ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+
+对象操作API (兼容Amazon S3):
+   Go:         https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
+   Java:       https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
+   Python:     https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
+   JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
+
+```
+
+### OS X
+
+
+ ```sh
+
+$ chmod 755 minio
+$ ./minio --help
+$ ./minio server ~/Photos
+
+端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
+AccessKey: USWUXHGYZQYFYFFIT3RE
+SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+区域:    us-east-1
+
+浏览器访问入口:
+   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
+
+命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
+   $ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+
+对象操作API (兼容Amazon S3):
+   Go:         https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
+   Java:       https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
+   Python:     https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
+   JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
+
+```
+
+### Microsoft Windows
+
+```sh
+
+C:\Users\Username\Downloads> minio.exe --help
+C:\Users\Username\Downloads> minio.exe server D:\Photos
+
+端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
+AccessKey: USWUXHGYZQYFYFFIT3RE
+SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+区域:    us-east-1
+
+浏览器访问入口:
+   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
+
+命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
+   $ mc.exe config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+
+对象操作API (兼容Amazon S3):
+   Go:         https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
+   Java:       https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
+   Python:     https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
+   JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
+
+
+```
+
+### Docker
+
+```sh
+
+$ docker pull minio/minio
+$ docker run -p 9000:9000 minio/minio
+
+```
+访问minio的docker入门指南获得更多内容 [here](https://docs.minio.io/docs/zh-CN/minio-docker-quickstart-guide)
+
+### FreeBSD
+
+```sh
+
+$ chmod 755 minio
+$ ./minio --help
+$ ./minio server ~/Photos
+
+端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
+AccessKey: USWUXHGYZQYFYFFIT3RE
+SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+区域:    us-east-1
+
+浏览器访问入口:
+   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
+
+命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
+   $ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+
+对象操作API (兼容Amazon S3):
+   Go:         https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
+   Java:       https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
+   Python:     https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
+   JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
+
+
+```
+请访问FreeBSD的官网指南获取更多详细信息[here](https://www.freebsd.org/doc/handbook/zfs-quickstart.html)
+
+## 3. 使用浏览器测试minio服务
+
+打开浏览器并输入 http://127.0.0.1:9000 查看在minio服务器上面的所有bucket
+
+![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
+
+
+## 4. 使用`mc`测试minio服务
+
+
+按照 [这个](https://docs.minio.io/docs/minio-client-quickstart-guide) 安装mc. 使用 `mc ls` 命令显示所有在minio服务上面的bucket.
+
+```sh
+
+$ mc ls myminio/
+[2015-08-05 08:13:22 IST]     0B andoria/
+[2015-08-05 06:14:26 IST]     0B deflector/
+[2015-08-05 08:13:11 IST]     0B ferenginar/
+[2016-03-08 14:56:35 IST]     0B jarjarbing/
+[2016-01-20 16:07:41 IST]     0B my.minio.io/
+
+```
+
+查看更多的例子请访问 [Minio Client Complete Guide](https://docs.minio.io/docs/zh-CN/minio-client-complete-guide).
+
+
+## 5. 更多内容
+
+- [Minio Erasure Code 快速入门](https://docs.minio.io/docs/zh-CN/minio-erasure-code-quickstart-guide)
+- [Minio Docker 快速入门](https://docs.minio.io/docs/zh-CN/minio-docker-quickstart-guide)
+- [使用`mc`测试 Minio Server](https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide)
+- [使用 `aws-cli` 测试 Minio Server](https://docs.minio.io/docs/zh-CN/aws-cli-with-minio)
+- [使用 `s3cmd` 测试 Minio Server](https://docs.minio.io/docs/zh-CN/s3cmd-with-minio)
+- [使用 `minio-go` SDK ce's测试 Minio Server](https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide)
+
+
+## 6. 给Minio项目贡献
+请按照Minio [贡献者指导手册](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)

accesslog-handler.go

@@ -1,120 +0,0 @@
-/*
- * Minimalist Object Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"encoding/json"
-	"net/http"
-	"net/url"
-	"os"
-	"time"
-
-	"github.com/minio/minio-xl/pkg/probe"
-)
-
-type accessLogHandler struct {
-	http.Handler
-	accessLogFile *os.File
-}
-
-// LogMessage is a serializable json log message
-type LogMessage struct {
-	StartTime     time.Time
-	Duration      time.Duration
-	StatusMessage string // human readable http status message
-	ContentLength string // human readable content length
-
-	// HTTP detailed message
-	HTTP struct {
-		ResponseHeaders http.Header
-		Request         struct {
-			Method     string
-			URL        *url.URL
-			Proto      string // "HTTP/1.0"
-			ProtoMajor int    // 1
-			ProtoMinor int    // 0
-			Header     http.Header
-			Host       string
-			Form       url.Values
-			PostForm   url.Values
-			Trailer    http.Header
-			RemoteAddr string
-			RequestURI string
-		}
-	}
-}
-
-func (h *accessLogHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	message, err := getLogMessage(w, req)
-	fatalIf(err.Trace(), "Unable to extract http message.", nil)
-	_, e := h.accessLogFile.Write(message)
-	fatalIf(probe.NewError(e), "Writing to log file failed.", nil)
-
-	h.Handler.ServeHTTP(w, req)
-}
-
-func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, *probe.Error) {
-	logMessage := &LogMessage{
-		StartTime: time.Now().UTC(),
-	}
-	// store lower level details
-	logMessage.HTTP.ResponseHeaders = w.Header()
-	logMessage.HTTP.Request = struct {
-		Method     string
-		URL        *url.URL
-		Proto      string // "HTTP/1.0"
-		ProtoMajor int    // 1
-		ProtoMinor int    // 0
-		Header     http.Header
-		Host       string
-		Form       url.Values
-		PostForm   url.Values
-		Trailer    http.Header
-		RemoteAddr string
-		RequestURI string
-	}{
-		Method:     req.Method,
-		URL:        req.URL,
-		Proto:      req.Proto,
-		ProtoMajor: req.ProtoMajor,
-		ProtoMinor: req.ProtoMinor,
-		Header:     req.Header,
-		Host:       req.Host,
-		Form:       req.Form,
-		PostForm:   req.PostForm,
-		Trailer:    req.Header,
-		RemoteAddr: req.RemoteAddr,
-		RequestURI: req.RequestURI,
-	}
-
-	// logMessage.HTTP.Request = req
-	logMessage.Duration = time.Now().UTC().Sub(logMessage.StartTime)
-	js, e := json.Marshal(logMessage)
-	if e != nil {
-		return nil, probe.NewError(e)
-	}
-	js = append(js, byte('\n')) // append a new line
-	return js, nil
-}
-
-// setAccessLogHandler logs requests
-func setAccessLogHandler(h http.Handler) http.Handler {
-	file, e := os.OpenFile("access.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
-	fatalIf(probe.NewError(e), "Unable to open access log.", nil)
-
-	return &accessLogHandler{Handler: h, accessLogFile: file}
-}

api-auth-utils.go

@@ -1,76 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"crypto/rand"
-	"encoding/base64"
-	"regexp"
-
-	"github.com/minio/minio-xl/pkg/probe"
-)
-
-const (
-	minioAccessID = 20
-	minioSecretID = 40
-)
-
-// isValidAccessKey - validate access key
-func isValidAccessKey(accessKeyID string) bool {
-	if accessKeyID == "" {
-		return true
-	}
-	regex := regexp.MustCompile("^[A-Z0-9\\-\\.\\_\\~]{20}$")
-	return regex.MatchString(accessKeyID)
-}
-
-// generateAccessKeyID - generate random alpha numeric value using only uppercase characters
-// takes input as size in integer
-func generateAccessKeyID() ([]byte, *probe.Error) {
-	alpha := make([]byte, minioAccessID)
-	if _, e := rand.Read(alpha); e != nil {
-		return nil, probe.NewError(e)
-	}
-	for i := 0; i < minioAccessID; i++ {
-		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
-	}
-	return alpha, nil
-}
-
-// generateSecretAccessKey - generate random base64 numeric value from a random seed.
-func generateSecretAccessKey() ([]byte, *probe.Error) {
-	rb := make([]byte, minioSecretID)
-	if _, e := rand.Read(rb); e != nil {
-		return nil, probe.NewError(e)
-	}
-	return []byte(base64.StdEncoding.EncodeToString(rb))[:minioSecretID], nil
-}
-
-// mustGenerateAccessKeyID - must generate random alpha numeric value using only uppercase characters
-// takes input as size in integer
-func mustGenerateAccessKeyID() []byte {
-	alpha, err := generateAccessKeyID()
-	fatalIf(err.Trace(), "Unable to generate accessKeyID.", nil)
-	return alpha
-}
-
-// mustGenerateSecretAccessKey - generate random base64 numeric value from a random seed.
-func mustGenerateSecretAccessKey() []byte {
-	secretKey, err := generateSecretAccessKey()
-	fatalIf(err.Trace(), "Unable to generate secretAccessKey.", nil)
-	return secretKey
-}

api-definitions.go

@@ -1,225 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "encoding/xml"
-
-// Limit number of objects in a given response.
-const (
-	maxObjectList = 1000
-)
-
-// LocationResponse - format for location response.
-type LocationResponse struct {
-	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
-	Location string   `xml:",chardata"`
-}
-
-// AccessControlPolicyResponse - format for get bucket acl response.
-type AccessControlPolicyResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
-
-	AccessControlList struct {
-		Grant []Grant
-	}
-	Owner Owner
-}
-
-// Grant container for grantee and permission.
-type Grant struct {
-	Grantee struct {
-		ID           string
-		DisplayName  string
-		EmailAddress string
-		Type         string
-		URI          string
-	}
-	Permission string
-}
-
-// ListObjectsResponse - format for list objects response.
-type ListObjectsResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
-
-	CommonPrefixes []*CommonPrefix
-	Contents       []*Object
-
-	Delimiter string
-
-	// Encoding type used to encode object keys in the response.
-	EncodingType string
-
-	// A flag that indicates whether or not ListObjects returned all of the results
-	// that satisfied the search criteria.
-	IsTruncated bool
-	Marker      string
-	MaxKeys     int
-	Name        string
-
-	// When response is truncated (the IsTruncated element value in the response
-	// is true), you can use the key name in this field as marker in the subsequent
-	// request to get next set of objects. Server lists objects in alphabetical
-	// order Note: This element is returned only if you have delimiter request parameter
-	// specified. If response does not include the NextMaker and it is truncated,
-	// you can use the value of the last Key in the response as the marker in the
-	// subsequent request to get the next set of object keys.
-	NextMarker string
-	Prefix     string
-}
-
-// Part container for part metadata.
-type Part struct {
-	PartNumber   int
-	ETag         string
-	LastModified string
-	Size         int64
-}
-
-// ListPartsResponse - format for list parts response.
-type ListPartsResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
-
-	Bucket   string
-	Key      string
-	UploadID string `xml:"UploadId"`
-
-	Initiator Initiator
-	Owner     Owner
-
-	// The class of storage used to store the object.
-	StorageClass string
-
-	PartNumberMarker     int
-	NextPartNumberMarker int
-	MaxParts             int
-	IsTruncated          bool
-
-	// List of parts.
-	Part []*Part
-}
-
-// ListMultipartUploadsResponse - format for list multipart uploads response.
-type ListMultipartUploadsResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
-
-	Bucket             string
-	KeyMarker          string
-	UploadIDMarker     string `xml:"UploadIdMarker"`
-	NextKeyMarker      string
-	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
-	EncodingType       string
-	MaxUploads         int
-	IsTruncated        bool
-	Upload             []*Upload
-	Prefix             string
-	Delimiter          string
-	CommonPrefixes     []*CommonPrefix
-}
-
-// ListBucketsResponse - format for list buckets response
-type ListBucketsResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
-	// Container for one or more buckets.
-	Buckets struct {
-		Bucket []*Bucket
-	} // Buckets are nested
-	Owner Owner
-}
-
-// Upload container for in progress multipart upload
-type Upload struct {
-	Key          string
-	UploadID     string `xml:"UploadId"`
-	Initiator    Initiator
-	Owner        Owner
-	StorageClass string
-	Initiated    string
-}
-
-// CommonPrefix container for prefix response in ListObjectsResponse
-type CommonPrefix struct {
-	Prefix string
-}
-
-// Bucket container for bucket metadata
-type Bucket struct {
-	Name         string
-	CreationDate string
-}
-
-// Object container for object metadata
-type Object struct {
-	ETag         string
-	Key          string
-	LastModified string
-	Size         int64
-
-	Owner Owner
-
-	// The class of storage used to store the object.
-	StorageClass string
-}
-
-// Initiator inherit from Owner struct, fields are same
-type Initiator Owner
-
-// Owner - bucket owner/principal
-type Owner struct {
-	ID          string
-	DisplayName string
-}
-
-// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
-type InitiateMultipartUploadResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
-
-	Bucket   string
-	Key      string
-	UploadID string `xml:"UploadId"`
-}
-
-// CompleteMultipartUploadResponse container for completed multipart upload response
-type CompleteMultipartUploadResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
-
-	Location string
-	Bucket   string
-	Key      string
-	ETag     string
-}
-
-// List of not implemented bucket queries
-var notimplementedBucketResourceNames = map[string]bool{
-	"policy":         true,
-	"cors":           true,
-	"lifecycle":      true,
-	"logging":        true,
-	"notification":   true,
-	"replication":    true,
-	"tagging":        true,
-	"versions":       true,
-	"requestPayment": true,
-	"versioning":     true,
-	"website":        true,
-}
-
-// List of not implemented object queries
-var notimplementedObjectResourceNames = map[string]bool{
-	"torrent": true,
-	"acl":     true,
-	"policy":  true,
-}

api-errors.go

@@ -1,268 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"encoding/xml"
-	"net/http"
-)
-
-// APIError structure
-type APIError struct {
-	Code           string
-	Description    string
-	HTTPStatusCode int
-}
-
-// APIErrorResponse - error response format
-type APIErrorResponse struct {
-	XMLName    xml.Name `xml:"Error" json:"-"`
-	Code       string
-	Message    string
-	Key        string
-	BucketName string
-	Resource   string
-	RequestID  string `xml:"RequestId"`
-	HostID     string `xml:"HostId"`
-}
-
-// Error codes, non exhaustive list - http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
-const (
-	AccessDenied = iota
-	BadDigest
-	BucketAlreadyExists
-	EntityTooSmall
-	EntityTooLarge
-	IncompleteBody
-	InternalError
-	InvalidAccessKeyID
-	InvalidBucketName
-	InvalidDigest
-	InvalidRange
-	InvalidMaxKeys
-	InvalidMaxUploads
-	InvalidMaxParts
-	InvalidPartNumberMarker
-	MalformedXML
-	MissingContentLength
-	MissingRequestBodyError
-	NoSuchBucket
-	NoSuchKey
-	NoSuchUpload
-	NotImplemented
-	RequestTimeTooSkewed
-	SignatureDoesNotMatch
-	MethodNotAllowed
-	InvalidPart
-	InvalidPartOrder
-	AuthorizationHeaderMalformed
-	MalformedPOSTRequest
-	SignatureVersionNotSupported
-	BucketNotEmpty
-	RootPathFull
-	ObjectExistsAsPrefix
-)
-
-// APIError code to Error structure map
-var errorCodeResponse = map[int]APIError{
-	InvalidMaxUploads: {
-		Code:           "InvalidArgument",
-		Description:    "Argument maxUploads must be an integer between 0 and 2147483647.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	InvalidMaxKeys: {
-		Code:           "InvalidArgument",
-		Description:    "Argument maxKeys must be an integer between 0 and 2147483647.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	InvalidMaxParts: {
-		Code:           "InvalidArgument",
-		Description:    "Argument maxParts must be an integer between 1 and 10000.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	InvalidPartNumberMarker: {
-		Code:           "InvalidArgument",
-		Description:    "Argument partNumberMarker must be an integer.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	AccessDenied: {
-		Code:           "AccessDenied",
-		Description:    "Access Denied.",
-		HTTPStatusCode: http.StatusForbidden,
-	},
-	BadDigest: {
-		Code:           "BadDigest",
-		Description:    "The Content-MD5 you specified did not match what we received.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	BucketAlreadyExists: {
-		Code:           "BucketAlreadyExists",
-		Description:    "The requested bucket name is not available.",
-		HTTPStatusCode: http.StatusConflict,
-	},
-	EntityTooSmall: {
-		Code:           "EntityTooSmall",
-		Description:    "Your proposed upload is smaller than the minimum allowed object size.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	EntityTooLarge: {
-		Code:           "EntityTooLarge",
-		Description:    "Your proposed upload exceeds the maximum allowed object size.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	IncompleteBody: {
-		Code:           "IncompleteBody",
-		Description:    "You did not provide the number of bytes specified by the Content-Length HTTP header.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	InternalError: {
-		Code:           "InternalError",
-		Description:    "We encountered an internal error, please try again.",
-		HTTPStatusCode: http.StatusInternalServerError,
-	},
-	InvalidAccessKeyID: {
-		Code:           "InvalidAccessKeyID",
-		Description:    "The access key ID you provided does not exist in our records.",
-		HTTPStatusCode: http.StatusForbidden,
-	},
-	InvalidBucketName: {
-		Code:           "InvalidBucketName",
-		Description:    "The specified bucket is not valid.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	InvalidDigest: {
-		Code:           "InvalidDigest",
-		Description:    "The Content-MD5 you specified is not valid.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	InvalidRange: {
-		Code:           "InvalidRange",
-		Description:    "The requested range cannot be satisfied.",
-		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
-	},
-	MalformedXML: {
-		Code:           "MalformedXML",
-		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	MissingContentLength: {
-		Code:           "MissingContentLength",
-		Description:    "You must provide the Content-Length HTTP header.",
-		HTTPStatusCode: http.StatusLengthRequired,
-	},
-	MissingRequestBodyError: {
-		Code:           "MissingRequestBodyError",
-		Description:    "Request body is empty.",
-		HTTPStatusCode: http.StatusLengthRequired,
-	},
-	NoSuchBucket: {
-		Code:           "NoSuchBucket",
-		Description:    "The specified bucket does not exist.",
-		HTTPStatusCode: http.StatusNotFound,
-	},
-	NoSuchKey: {
-		Code:           "NoSuchKey",
-		Description:    "The specified key does not exist.",
-		HTTPStatusCode: http.StatusNotFound,
-	},
-	NoSuchUpload: {
-		Code:           "NoSuchUpload",
-		Description:    "The specified multipart upload does not exist.",
-		HTTPStatusCode: http.StatusNotFound,
-	},
-	NotImplemented: {
-		Code:           "NotImplemented",
-		Description:    "A header you provided implies functionality that is not implemented.",
-		HTTPStatusCode: http.StatusNotImplemented,
-	},
-	RequestTimeTooSkewed: {
-		Code:           "RequestTimeTooSkewed",
-		Description:    "The difference between the request time and the server's time is too large.",
-		HTTPStatusCode: http.StatusForbidden,
-	},
-	SignatureDoesNotMatch: {
-		Code:           "SignatureDoesNotMatch",
-		Description:    "The request signature we calculated does not match the signature you provided.",
-		HTTPStatusCode: http.StatusForbidden,
-	},
-	MethodNotAllowed: {
-		Code:           "MethodNotAllowed",
-		Description:    "The specified method is not allowed against this resource.",
-		HTTPStatusCode: http.StatusMethodNotAllowed,
-	},
-	InvalidPart: {
-		Code:           "InvalidPart",
-		Description:    "One or more of the specified parts could not be found.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	InvalidPartOrder: {
-		Code:           "InvalidPartOrder",
-		Description:    "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	AuthorizationHeaderMalformed: {
-		Code:           "AuthorizationHeaderMalformed",
-		Description:    "The authorization header is malformed; the region is wrong; expecting 'us-east-1'.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	MalformedPOSTRequest: {
-		Code:           "MalformedPOSTRequest",
-		Description:    "The body of your POST request is not well-formed multipart/form-data.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	SignatureVersionNotSupported: {
-		Code:           "InvalidRequest",
-		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	BucketNotEmpty: {
-		Code:           "BucketNotEmpty",
-		Description:    "The bucket you tried to delete is not empty.",
-		HTTPStatusCode: http.StatusConflict,
-	},
-	RootPathFull: {
-		Code:           "RootPathFull",
-		Description:    "Root path has reached its minimum free disk threshold. Please delete few objects to proceed.",
-		HTTPStatusCode: http.StatusInternalServerError,
-	},
-	ObjectExistsAsPrefix: {
-		Code:           "ObjectExistsAsPrefix",
-		Description:    "An object already exists as your prefix, choose a different prefix to proceed.",
-		HTTPStatusCode: http.StatusConflict,
-	},
-}
-
-// errorCodeError provides errorCode to Error. It returns empty if the code provided is unknown
-func getErrorCode(code int) APIError {
-	return errorCodeResponse[code]
-}
-
-// getErrorResponse gets in standard error and resource value and
-// provides a encodable populated response values
-func getErrorResponse(err APIError, resource string) APIErrorResponse {
-	var data = APIErrorResponse{}
-	data.Code = err.Code
-	data.Message = err.Description
-	if resource != "" {
-		data.Resource = resource
-	}
-	// TODO implement this in future
-	data.RequestID = "3L137"
-	data.HostID = "3L137"
-
-	return data
-}

api-headers.go

@@ -1,102 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"bytes"
-	"crypto/rand"
-	"encoding/xml"
-	"net/http"
-	"runtime"
-	"strconv"
-
-	"github.com/minio/minio/pkg/fs"
-)
-
-//// helpers
-
-// Static alphaNumeric table used for generating unique request ids
-var alphaNumericTable = []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
-
-// generateRequestID - Generate request id
-func generateRequestID() []byte {
-	alpha := make([]byte, 16)
-	rand.Read(alpha)
-	for i := 0; i < 16; i++ {
-		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
-	}
-	return alpha
-}
-
-// Write http common headers
-func setCommonHeaders(w http.ResponseWriter) {
-	// Set unique request ID for each reply.
-	w.Header().Set("X-Amz-Request-Id", string(generateRequestID()))
-	w.Header().Set("Server", ("Minio/" + minioReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"))
-	w.Header().Set("Accept-Ranges", "bytes")
-}
-
-// Write error response headers
-func encodeErrorResponse(response interface{}) []byte {
-	var bytesBuffer bytes.Buffer
-	// write common headers
-	bytesBuffer.WriteString(xml.Header)
-	e := xml.NewEncoder(&bytesBuffer)
-	e.Encode(response)
-	return bytesBuffer.Bytes()
-}
-
-// Write object header
-func setObjectHeaders(w http.ResponseWriter, metadata fs.ObjectMetadata, contentRange *httpRange) {
-	// set common headers
-	if contentRange != nil {
-		if contentRange.length > 0 {
-			w.Header().Set("Content-Length", strconv.FormatInt(contentRange.length, 10))
-			setCommonHeaders(w)
-		} else {
-			w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
-			setCommonHeaders(w)
-		}
-	} else {
-		w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
-		setCommonHeaders(w)
-	}
-	// set object headers
-	lastModified := metadata.Created.Format(http.TimeFormat)
-	// object related headers
-	w.Header().Set("Content-Type", metadata.ContentType)
-	if metadata.MD5 != "" {
-		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
-	}
-	w.Header().Set("Last-Modified", lastModified)
-
-	// set content range
-	if contentRange != nil {
-		if contentRange.start > 0 || contentRange.length > 0 {
-			w.Header().Set("Content-Range", contentRange.String())
-			w.WriteHeader(http.StatusPartialContent)
-		}
-	}
-}
-
-func encodeSuccessResponse(response interface{}) []byte {
-	var bytesBuffer bytes.Buffer
-	bytesBuffer.WriteString(xml.Header)
-	e := xml.NewEncoder(&bytesBuffer)
-	e.Encode(response)
-	return bytesBuffer.Bytes()
-}

api-resources.go

@@ -1,54 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"net/url"
-	"strconv"
-
-	"github.com/minio/minio/pkg/fs"
-)
-
-// parse bucket url queries
-func getBucketResources(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
-	prefix = values.Get("prefix")
-	marker = values.Get("marker")
-	delimiter = values.Get("delimiter")
-	maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
-	encodingType = values.Get("encoding-type")
-	return
-}
-
-// part bucket url queries for ?uploads
-func getBucketMultipartResources(values url.Values) (v fs.BucketMultipartResourcesMetadata) {
-	v.Prefix = values.Get("prefix")
-	v.KeyMarker = values.Get("key-marker")
-	v.MaxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
-	v.Delimiter = values.Get("delimiter")
-	v.EncodingType = values.Get("encoding-type")
-	v.UploadIDMarker = values.Get("upload-id-marker")
-	return
-}
-
-// parse object url queries
-func getObjectResources(values url.Values) (v fs.ObjectResourcesMetadata) {
-	v.UploadID = values.Get("uploadId")
-	v.PartNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
-	v.MaxParts, _ = strconv.Atoi(values.Get("max-parts"))
-	v.EncodingType = values.Get("encoding-type")
-	return
-}

api-response.go

@@ -1,246 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"net/http"
-
-	"github.com/minio/minio/pkg/fs"
-)
-
-// Reply date format
-const (
-	rfcFormat = "2006-01-02T15:04:05.000Z"
-)
-
-// takes an array of Bucketmetadata information for serialization
-// input:
-// array of bucket metadata
-//
-// output:
-// populated struct that can be serialized to match xml and json api spec output
-func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsResponse {
-	var listbuckets []*Bucket
-	var data = ListBucketsResponse{}
-	var owner = Owner{}
-
-	owner.ID = "minio"
-	owner.DisplayName = "minio"
-
-	for _, bucket := range buckets {
-		var listbucket = &Bucket{}
-		listbucket.Name = bucket.Name
-		listbucket.CreationDate = bucket.Created.Format(rfcFormat)
-		listbuckets = append(listbuckets, listbucket)
-	}
-
-	data.Owner = owner
-	data.Buckets.Bucket = listbuckets
-
-	return data
-}
-
-// generates an AccessControlPolicy response for the said ACL.
-func generateAccessControlPolicyResponse(acl fs.BucketACL) AccessControlPolicyResponse {
-	accessCtrlPolicyResponse := AccessControlPolicyResponse{}
-	accessCtrlPolicyResponse.Owner = Owner{
-		ID:          "minio",
-		DisplayName: "minio",
-	}
-	defaultGrant := Grant{}
-	defaultGrant.Grantee.ID = "minio"
-	defaultGrant.Grantee.DisplayName = "minio"
-	defaultGrant.Permission = "FULL_CONTROL"
-	accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, defaultGrant)
-	switch {
-	case acl.IsPublicRead():
-		publicReadGrant := Grant{}
-		publicReadGrant.Grantee.ID = "minio"
-		publicReadGrant.Grantee.DisplayName = "minio"
-		publicReadGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers"
-		publicReadGrant.Permission = "READ"
-		accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadGrant)
-	case acl.IsPublicReadWrite():
-		publicReadGrant := Grant{}
-		publicReadGrant.Grantee.ID = "minio"
-		publicReadGrant.Grantee.DisplayName = "minio"
-		publicReadGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers"
-		publicReadGrant.Permission = "READ"
-		publicReadWriteGrant := Grant{}
-		publicReadWriteGrant.Grantee.ID = "minio"
-		publicReadWriteGrant.Grantee.DisplayName = "minio"
-		publicReadWriteGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers"
-		publicReadWriteGrant.Permission = "WRITE"
-		accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadGrant)
-		accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadWriteGrant)
-	}
-	return accessCtrlPolicyResponse
-}
-
-// generates an ListObjects response for the said bucket with other enumerated options.
-func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp fs.ListObjectsResult) ListObjectsResponse {
-	var contents []*Object
-	var prefixes []*CommonPrefix
-	var owner = Owner{}
-	var data = ListObjectsResponse{}
-
-	owner.ID = "minio"
-	owner.DisplayName = "minio"
-
-	for _, object := range resp.Objects {
-		var content = &Object{}
-		if object.Object == "" {
-			continue
-		}
-		content.Key = object.Object
-		content.LastModified = object.Created.Format(rfcFormat)
-		if object.MD5 != "" {
-			content.ETag = "\"" + object.MD5 + "\""
-		}
-		content.Size = object.Size
-		content.StorageClass = "STANDARD"
-		content.Owner = owner
-		contents = append(contents, content)
-	}
-	// TODO - support EncodingType in xml decoding
-	data.Name = bucket
-	data.Contents = contents
-
-	data.Prefix = prefix
-	data.Marker = marker
-	data.Delimiter = delimiter
-	data.MaxKeys = maxKeys
-
-	data.NextMarker = resp.NextMarker
-	data.IsTruncated = resp.IsTruncated
-	for _, prefix := range resp.Prefixes {
-		var prefixItem = &CommonPrefix{}
-		prefixItem.Prefix = prefix
-		prefixes = append(prefixes, prefixItem)
-	}
-	data.CommonPrefixes = prefixes
-	return data
-}
-
-// generateInitiateMultipartUploadResponse
-func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse {
-	return InitiateMultipartUploadResponse{
-		Bucket:   bucket,
-		Key:      key,
-		UploadID: uploadID,
-	}
-}
-
-// generateCompleteMultipartUploadResponse
-func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
-	return CompleteMultipartUploadResponse{
-		Location: location,
-		Bucket:   bucket,
-		Key:      key,
-		ETag:     etag,
-	}
-}
-
-// generateListPartsResult
-func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPartsResponse {
-	// TODO - support EncodingType in xml decoding
-	listPartsResponse := ListPartsResponse{}
-	listPartsResponse.Bucket = objectMetadata.Bucket
-	listPartsResponse.Key = objectMetadata.Object
-	listPartsResponse.UploadID = objectMetadata.UploadID
-	listPartsResponse.StorageClass = "STANDARD"
-	listPartsResponse.Initiator.ID = "minio"
-	listPartsResponse.Initiator.DisplayName = "minio"
-	listPartsResponse.Owner.ID = "minio"
-	listPartsResponse.Owner.DisplayName = "minio"
-
-	listPartsResponse.MaxParts = objectMetadata.MaxParts
-	listPartsResponse.PartNumberMarker = objectMetadata.PartNumberMarker
-	listPartsResponse.IsTruncated = objectMetadata.IsTruncated
-	listPartsResponse.NextPartNumberMarker = objectMetadata.NextPartNumberMarker
-
-	listPartsResponse.Part = make([]*Part, len(objectMetadata.Part))
-	for _, part := range objectMetadata.Part {
-		newPart := &Part{}
-		newPart.PartNumber = part.PartNumber
-		newPart.ETag = "\"" + part.ETag + "\""
-		newPart.Size = part.Size
-		newPart.LastModified = part.LastModified.Format(rfcFormat)
-		listPartsResponse.Part = append(listPartsResponse.Part, newPart)
-	}
-	return listPartsResponse
-}
-
-// generateListMultipartUploadsResponse
-func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse {
-	listMultipartUploadsResponse := ListMultipartUploadsResponse{}
-	listMultipartUploadsResponse.Bucket = bucket
-	listMultipartUploadsResponse.Delimiter = metadata.Delimiter
-	listMultipartUploadsResponse.IsTruncated = metadata.IsTruncated
-	listMultipartUploadsResponse.EncodingType = metadata.EncodingType
-	listMultipartUploadsResponse.Prefix = metadata.Prefix
-	listMultipartUploadsResponse.KeyMarker = metadata.KeyMarker
-	listMultipartUploadsResponse.NextKeyMarker = metadata.NextKeyMarker
-	listMultipartUploadsResponse.MaxUploads = metadata.MaxUploads
-	listMultipartUploadsResponse.NextUploadIDMarker = metadata.NextUploadIDMarker
-	listMultipartUploadsResponse.UploadIDMarker = metadata.UploadIDMarker
-
-	listMultipartUploadsResponse.Upload = make([]*Upload, len(metadata.Upload))
-	for _, upload := range metadata.Upload {
-		newUpload := &Upload{}
-		newUpload.UploadID = upload.UploadID
-		newUpload.Key = upload.Object
-		newUpload.Initiated = upload.Initiated.Format(rfcFormat)
-		listMultipartUploadsResponse.Upload = append(listMultipartUploadsResponse.Upload, newUpload)
-	}
-	return listMultipartUploadsResponse
-}
-
-// writeSuccessResponse write success headers and response if any.
-func writeSuccessResponse(w http.ResponseWriter, response []byte) {
-	setCommonHeaders(w)
-	if response == nil {
-		w.WriteHeader(http.StatusOK)
-		return
-	}
-	w.Write(response)
-	w.(http.Flusher).Flush()
-}
-
-// writeSuccessNoContent write success headers with http status 204
-func writeSuccessNoContent(w http.ResponseWriter) {
-	setCommonHeaders(w)
-	w.WriteHeader(http.StatusNoContent)
-}
-
-// writeErrorRespone write error headers
-func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorType int, resource string) {
-	error := getErrorCode(errorType)
-	// generate error response
-	errorResponse := getErrorResponse(error, resource)
-	encodedErrorResponse := encodeErrorResponse(errorResponse)
-	// set common headers
-	setCommonHeaders(w)
-	// write Header
-	w.WriteHeader(error.HTTPStatusCode)
-	// HEAD should have no body, do not attempt to write to it
-	if req.Method != "HEAD" {
-		// write error body
-		w.Write(encodedErrorResponse)
-		w.(http.Flusher).Flush()
-	}
-}

api-signature.go

@@ -1,304 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"bytes"
-	"encoding/base64"
-	"io"
-	"io/ioutil"
-	"mime/multipart"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/minio/minio-xl/pkg/probe"
-	"github.com/minio/minio/pkg/fs"
-)
-
-const (
-	authHeaderPrefix = "AWS4-HMAC-SHA256"
-	iso8601Format    = "20060102T150405Z"
-)
-
-// getCredentialsFromAuth parse credentials tag from authorization value
-func getCredentialsFromAuth(authValue string) ([]string, *probe.Error) {
-	if authValue == "" {
-		return nil, probe.NewError(errMissingAuthHeaderValue)
-	}
-	// replace all spaced strings
-	authValue = strings.Replace(authValue, " ", "", -1)
-	if !strings.HasPrefix(authValue, authHeaderPrefix) {
-		return nil, probe.NewError(errMissingFieldsAuthHeader)
-	}
-	if !strings.HasPrefix(strings.TrimPrefix(authValue, authHeaderPrefix), "Credential") {
-		return nil, probe.NewError(errInvalidAuthHeaderPrefix)
-	}
-	authValue = strings.TrimPrefix(authValue, authHeaderPrefix)
-	authFields := strings.Split(strings.TrimSpace(authValue), ",")
-	if len(authFields) != 3 {
-		return nil, probe.NewError(errInvalidAuthHeaderValue)
-	}
-	credentials := strings.Split(strings.TrimSpace(authFields[0]), "=")
-	if len(credentials) != 2 {
-		return nil, probe.NewError(errMissingFieldsCredentialTag)
-	}
-	credentialElements := strings.Split(strings.TrimSpace(credentials[1]), "/")
-	if len(credentialElements) != 5 {
-		return nil, probe.NewError(errCredentialTagMalformed)
-	}
-	return credentialElements, nil
-}
-
-func getSignatureFromAuth(authHeaderValue string) (string, *probe.Error) {
-	authValue := strings.TrimPrefix(authHeaderValue, authHeaderPrefix)
-	authFields := strings.Split(strings.TrimSpace(authValue), ",")
-	if len(authFields) != 3 {
-		return "", probe.NewError(errInvalidAuthHeaderValue)
-	}
-	if len(strings.Split(strings.TrimSpace(authFields[2]), "=")) != 2 {
-		return "", probe.NewError(errMissingFieldsSignatureTag)
-	}
-	signature := strings.Split(strings.TrimSpace(authFields[2]), "=")[1]
-	return signature, nil
-}
-
-func getSignedHeadersFromAuth(authHeaderValue string) ([]string, *probe.Error) {
-	authValue := strings.TrimPrefix(authHeaderValue, authHeaderPrefix)
-	authFields := strings.Split(strings.TrimSpace(authValue), ",")
-	if len(authFields) != 3 {
-		return nil, probe.NewError(errInvalidAuthHeaderValue)
-	}
-	if len(strings.Split(strings.TrimSpace(authFields[1]), "=")) != 2 {
-		return nil, probe.NewError(errMissingFieldsSignedHeadersTag)
-	}
-	signedHeaders := strings.Split(strings.Split(strings.TrimSpace(authFields[1]), "=")[1], ";")
-	return signedHeaders, nil
-}
-
-// verify if region value is valid with configured minioRegion.
-func isValidRegion(region string, minioRegion string) *probe.Error {
-	if minioRegion == "" {
-		minioRegion = "us-east-1"
-	}
-	if region != minioRegion && region != "US" {
-		return probe.NewError(errInvalidRegion)
-	}
-	return nil
-}
-
-// stripRegion - strip only region from auth header.
-func stripRegion(authHeaderValue string) (string, *probe.Error) {
-	credentialElements, err := getCredentialsFromAuth(authHeaderValue)
-	if err != nil {
-		return "", err.Trace(authHeaderValue)
-	}
-	region := credentialElements[2]
-	return region, nil
-}
-
-// stripAccessKeyID - strip only access key id from auth header.
-func stripAccessKeyID(authHeaderValue string) (string, *probe.Error) {
-	credentialElements, err := getCredentialsFromAuth(authHeaderValue)
-	if err != nil {
-		return "", err.Trace()
-	}
-	accessKeyID := credentialElements[0]
-	if !isValidAccessKey(accessKeyID) {
-		return "", probe.NewError(errAccessKeyIDInvalid)
-	}
-	return accessKeyID, nil
-}
-
-// initSignatureV4 initializing signature verification.
-func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
-	// strip auth from authorization header.
-	authHeaderValue := req.Header.Get("Authorization")
-
-	config, err := loadConfigV2()
-	if err != nil {
-		return nil, err.Trace()
-	}
-
-	region, err := stripRegion(authHeaderValue)
-	if err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-
-	if err = isValidRegion(region, config.Credentials.Region); err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-
-	accessKeyID, err := stripAccessKeyID(authHeaderValue)
-	if err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-	signature, err := getSignatureFromAuth(authHeaderValue)
-	if err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-	signedHeaders, err := getSignedHeadersFromAuth(authHeaderValue)
-	if err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-	if config.Credentials.AccessKeyID == accessKeyID {
-		signature := &fs.Signature{
-			AccessKeyID:     config.Credentials.AccessKeyID,
-			SecretAccessKey: config.Credentials.SecretAccessKey,
-			Region:          region,
-			Signature:       signature,
-			SignedHeaders:   signedHeaders,
-			Request:         req,
-		}
-		return signature, nil
-	}
-	return nil, probe.NewError(errAccessKeyIDInvalid)
-}
-
-func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]string, *probe.Error) {
-	/// HTML Form values
-	formValues := make(map[string]string)
-	filePart := new(bytes.Buffer)
-	var e error
-	for e == nil {
-		var part *multipart.Part
-		part, e = reader.NextPart()
-		if part != nil {
-			if part.FileName() == "" {
-				buffer, e := ioutil.ReadAll(part)
-				if e != nil {
-					return nil, nil, probe.NewError(e)
-				}
-				formValues[http.CanonicalHeaderKey(part.FormName())] = string(buffer)
-			} else {
-				if _, e := io.Copy(filePart, part); e != nil {
-					return nil, nil, probe.NewError(e)
-				}
-			}
-		}
-	}
-	return filePart, formValues, nil
-}
-
-func applyPolicy(formValues map[string]string) *probe.Error {
-	if formValues["X-Amz-Algorithm"] != "AWS4-HMAC-SHA256" {
-		return probe.NewError(errUnsupportedAlgorithm)
-	}
-	/// Decoding policy
-	policyBytes, e := base64.StdEncoding.DecodeString(formValues["Policy"])
-	if e != nil {
-		return probe.NewError(e)
-	}
-	postPolicyForm, err := fs.ParsePostPolicyForm(string(policyBytes))
-	if err != nil {
-		return err.Trace()
-	}
-	if !postPolicyForm.Expiration.After(time.Now().UTC()) {
-		return probe.NewError(errPolicyAlreadyExpired)
-	}
-	if postPolicyForm.Conditions.Policies["$bucket"].Operator == "eq" {
-		if formValues["Bucket"] != postPolicyForm.Conditions.Policies["$bucket"].Value {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$x-amz-date"].Operator == "eq" {
-		if formValues["X-Amz-Date"] != postPolicyForm.Conditions.Policies["$x-amz-date"].Value {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$Content-Type"].Operator == "starts-with" {
-		if !strings.HasPrefix(formValues["Content-Type"], postPolicyForm.Conditions.Policies["$Content-Type"].Value) {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$Content-Type"].Operator == "eq" {
-		if formValues["Content-Type"] != postPolicyForm.Conditions.Policies["$Content-Type"].Value {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$key"].Operator == "starts-with" {
-		if !strings.HasPrefix(formValues["Key"], postPolicyForm.Conditions.Policies["$key"].Value) {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$key"].Operator == "eq" {
-		if formValues["Key"] != postPolicyForm.Conditions.Policies["$key"].Value {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	return nil
-}
-
-// initPostPresignedPolicyV4 initializing post policy signature verification
-func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *probe.Error) {
-	credentialElements := strings.Split(strings.TrimSpace(formValues["X-Amz-Credential"]), "/")
-	if len(credentialElements) != 5 {
-		return nil, probe.NewError(errCredentialTagMalformed)
-	}
-	accessKeyID := credentialElements[0]
-	if !isValidAccessKey(accessKeyID) {
-		return nil, probe.NewError(errAccessKeyIDInvalid)
-	}
-	config, err := loadConfigV2()
-	if err != nil {
-		return nil, err.Trace()
-	}
-	region := credentialElements[2]
-	if config.Credentials.AccessKeyID == accessKeyID {
-		signature := &fs.Signature{
-			AccessKeyID:     config.Credentials.AccessKeyID,
-			SecretAccessKey: config.Credentials.SecretAccessKey,
-			Region:          region,
-			Signature:       formValues["X-Amz-Signature"],
-			PresignedPolicy: formValues["Policy"],
-		}
-		return signature, nil
-	}
-	return nil, probe.NewError(errAccessKeyIDInvalid)
-}
-
-// initPresignedSignatureV4 initializing presigned signature verification
-func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
-	credentialElements := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-Credential")), "/")
-	if len(credentialElements) != 5 {
-		return nil, probe.NewError(errCredentialTagMalformed)
-	}
-	accessKeyID := credentialElements[0]
-	if !isValidAccessKey(accessKeyID) {
-		return nil, probe.NewError(errAccessKeyIDInvalid)
-	}
-	config, err := loadConfigV2()
-	if err != nil {
-		return nil, err.Trace()
-	}
-	region := credentialElements[2]
-	signedHeaders := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-SignedHeaders")), ";")
-	signature := strings.TrimSpace(req.URL.Query().Get("X-Amz-Signature"))
-	if config.Credentials.AccessKeyID == accessKeyID {
-		signature := &fs.Signature{
-			AccessKeyID:     config.Credentials.AccessKeyID,
-			SecretAccessKey: config.Credentials.SecretAccessKey,
-			Region:          region,
-			Signature:       signature,
-			SignedHeaders:   signedHeaders,
-			Presigned:       true,
-			Request:         req,
-		}
-		return signature, nil
-	}
-	return nil, probe.NewError(errAccessKeyIDInvalid)
-}

appveyor.yml

@@ -4,37 +4,51 @@ version: "{build}"
 # Operating system (build VM template)
 os: Windows Server 2012 R2
 
+# Platform.
+platform: x64
+
 clone_folder: c:\gopath\src\github.com\minio\minio
 
-# environment variables
+# Environment variables
 environment:
   GOPATH: c:\gopath
   GO15VENDOREXPERIMENT: 1
-  UI_ASSETS: ui-assets.go
-  UI_ASSETS_ARMOR: ui-assets.asc
 
 # scripts that run after cloning repository
 install:
   - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
   - go version
   - go env
+  - python --version
 
-# to run your custom scripts instead of automatic MSBuild
+# To run your custom scripts instead of automatic MSBuild
 build_script:
-  - curl -o ui-assets.go -L https://dl.minio.io/assets/server/%UI_ASSETS%
-  - curl -o ui-assets.asc -L https://dl.minio.io/assets/server/%UI_ASSETS_ARMOR%
-  - gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728
-  - gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS%  
-  - go test .
-  - go test -race .
-  - go test github.com/minio/minio/pkg...
-  - go test -race github.com/minio/minio/pkg...
+  # Compile
+  - appveyor AddCompilationMessage "Starting Compile"
+  - cd c:\gopath\src\github.com\minio\minio
   - go run buildscripts/gen-ldflags.go > temp.txt
   - set /p BUILD_LDFLAGS=<temp.txt
   - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
+  - appveyor AddCompilationMessage "Compile Success"
 
-# to disable automatic tests
-test: off
+# To run your custom scripts instead of automatic tests
+test_script:
+  # Unit tests
+  - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
+  - mkdir build\coverage
+  - go test -race github.com/minio/minio/cmd...
+  - go test -race github.com/minio/minio/pkg...
+  - go test -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
+
+after_test:
+  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
+  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
+  - ps: Push-AppveyorArtifact build\coverage\coverage.html
+  # Upload coverage report.
+  - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
+  - pip install codecov
+  - codecov -X gcov -f "build\coverage\coverage.txt"
 
 # to disable deployment
 deploy: off

bucket-acl.go

@@ -1,71 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "net/http"
-
-// Please read for more information - http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
-//
-// Here We are only supporting 'acl's through request headers not through their request body
-// http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#setting-acls
-
-// Minio only supports three types for now i.e 'private, public-read, public-read-write'
-
-// ACLType - different acl types
-type ACLType int
-
-const (
-	unsupportedACLType ACLType = iota
-	privateACLType
-	publicReadACLType
-	publicReadWriteACLType
-)
-
-// Get acl type requested from 'x-amz-acl' header
-func getACLType(req *http.Request) ACLType {
-	aclHeader := req.Header.Get("x-amz-acl")
-	if aclHeader != "" {
-		switch {
-		case aclHeader == "private":
-			return privateACLType
-		case aclHeader == "public-read":
-			return publicReadACLType
-		case aclHeader == "public-read-write":
-			return publicReadWriteACLType
-		default:
-			return unsupportedACLType
-		}
-	}
-	// make it default private
-	return privateACLType
-}
-
-// ACL type to human readable string
-func getACLTypeString(acl ACLType) string {
-	switch acl {
-	case privateACLType:
-		return "private"
-	case publicReadACLType:
-		return "public-read"
-	case publicReadWriteACLType:
-		return "public-read-write"
-	case unsupportedACLType:
-		return ""
-	default:
-		return "private"
-	}
-}

bucket-handlers.go

@@ -1,490 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"encoding/hex"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/minio-xl/pkg/crypto/sha256"
-	"github.com/minio/minio-xl/pkg/probe"
-	"github.com/minio/minio/pkg/fs"
-)
-
-// GetBucketLocationHandler - GET Bucket location.
-// -------------------------
-// This operation returns bucket location.
-func (api CloudStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-		return
-	}
-
-	_, err := api.Filesystem.GetBucketMetadata(bucket)
-	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-
-	// TODO: Location value for LocationResponse is deliberately not used, until
-	//       we bring in a mechanism of configurable regions. For the time being
-	//       default region is empty i.e 'us-east-1'.
-	encodedSuccessResponse := encodeSuccessResponse(LocationResponse{}) // generate response
-	setCommonHeaders(w)                                                 // write headers
-	writeSuccessResponse(w, encodedSuccessResponse)
-}
-
-// ListMultipartUploadsHandler - GET Bucket (List Multipart uploads)
-// -------------------------
-// This operation lists in-progress multipart uploads. An in-progress
-// multipart upload is a multipart upload that has been initiated,
-// using the Initiate Multipart Upload request, but has not yet been
-// completed or aborted. This operation returns at most 1,000 multipart
-// uploads in the response.
-//
-func (api CloudStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-		return
-	}
-
-	resources := getBucketMultipartResources(req.URL.Query())
-	if resources.MaxUploads < 0 {
-		writeErrorResponse(w, req, InvalidMaxUploads, req.URL.Path)
-		return
-	}
-	if resources.MaxUploads == 0 {
-		resources.MaxUploads = maxObjectList
-	}
-
-	resources, err := api.Filesystem.ListMultipartUploads(bucket, resources)
-	if err != nil {
-		errorIf(err.Trace(), "ListMultipartUploads failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-	// generate response
-	response := generateListMultipartUploadsResponse(bucket, resources)
-	encodedSuccessResponse := encodeSuccessResponse(response)
-	// write headers.
-	setCommonHeaders(w)
-	// write success response.
-	writeSuccessResponse(w, encodedSuccessResponse)
-}
-
-// ListObjectsHandler - GET Bucket (List Objects)
-// -------------------------
-// This implementation of the GET operation returns some or all (up to 1000)
-// of the objects in a bucket. You can use the request parameters as selection
-// criteria to return a subset of the objects in a bucket.
-//
-func (api CloudStorageAPI) ListObjectsHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		if api.Filesystem.IsPrivateBucket(bucket) {
-			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-			return
-		}
-	}
-
-	// TODO handle encoding type.
-	prefix, marker, delimiter, maxkeys, _ := getBucketResources(req.URL.Query())
-	if maxkeys < 0 {
-		writeErrorResponse(w, req, InvalidMaxKeys, req.URL.Path)
-		return
-	}
-	if maxkeys == 0 {
-		maxkeys = maxObjectList
-	}
-
-	listResp, err := api.Filesystem.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
-	if err == nil {
-		// generate response
-		response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listResp)
-		encodedSuccessResponse := encodeSuccessResponse(response)
-		// Write headers
-		setCommonHeaders(w)
-		// Write success response.
-		writeSuccessResponse(w, encodedSuccessResponse)
-		return
-	}
-	switch err.ToGoError().(type) {
-	case fs.BucketNameInvalid:
-		writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
-	case fs.BucketNotFound:
-		writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-	case fs.ObjectNotFound:
-		writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
-	case fs.ObjectNameInvalid:
-		writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
-	default:
-		errorIf(err.Trace(), "ListObjects failed.", nil)
-		writeErrorResponse(w, req, InternalError, req.URL.Path)
-	}
-}
-
-// ListBucketsHandler - GET Service
-// -----------
-// This implementation of the GET operation returns a list of all buckets
-// owned by the authenticated sender of the request.
-func (api CloudStorageAPI) ListBucketsHandler(w http.ResponseWriter, req *http.Request) {
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-		return
-	}
-
-	buckets, err := api.Filesystem.ListBuckets()
-	if err == nil {
-		// generate response
-		response := generateListBucketsResponse(buckets)
-		encodedSuccessResponse := encodeSuccessResponse(response)
-		// write headers
-		setCommonHeaders(w)
-		// write response
-		writeSuccessResponse(w, encodedSuccessResponse)
-		return
-	}
-	errorIf(err.Trace(), "ListBuckets failed.", nil)
-	writeErrorResponse(w, req, InternalError, req.URL.Path)
-}
-
-// PutBucketHandler - PUT Bucket
-// ----------
-// This implementation of the PUT operation creates a new bucket for authenticated request
-func (api CloudStorageAPI) PutBucketHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-		return
-	}
-
-	// read from 'x-amz-acl'
-	aclType := getACLType(req)
-	if aclType == unsupportedACLType {
-		writeErrorResponse(w, req, NotImplemented, req.URL.Path)
-		return
-	}
-
-	var signature *fs.Signature
-	// Init signature V4 verification
-	if isRequestSignatureV4(req) {
-		var err *probe.Error
-		signature, err = initSignatureV4(req)
-		if err != nil {
-			switch err.ToGoError() {
-			case errInvalidRegion:
-				errorIf(err.Trace(), "Unknown region in authorization header.", nil)
-				writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
-				return
-			case errAccessKeyIDInvalid:
-				errorIf(err.Trace(), "Invalid access key id.", nil)
-				writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
-				return
-			default:
-				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
-				writeErrorResponse(w, req, InternalError, req.URL.Path)
-				return
-			}
-		}
-	}
-
-	// if body of request is non-nil then check for validity of Content-Length
-	if req.Body != nil {
-		/// if Content-Length is unknown/missing, deny the request
-		if req.ContentLength == -1 && !contains(req.TransferEncoding, "chunked") {
-			writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
-			return
-		}
-		if signature != nil {
-			locationBytes, e := ioutil.ReadAll(req.Body)
-			if e != nil {
-				errorIf(probe.NewError(e), "MakeBucket failed.", nil)
-				writeErrorResponse(w, req, InternalError, req.URL.Path)
-				return
-			}
-			sh := sha256.New()
-			sh.Write(locationBytes)
-			ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
-			if err != nil {
-				errorIf(err.Trace(), "MakeBucket failed.", nil)
-				writeErrorResponse(w, req, InternalError, req.URL.Path)
-				return
-			}
-			if !ok {
-				writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
-				return
-			}
-		}
-	}
-
-	err := api.Filesystem.MakeBucket(bucket, getACLTypeString(aclType))
-	if err != nil {
-		errorIf(err.Trace(), "MakeBucket failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
-		case fs.BucketExists:
-			writeErrorResponse(w, req, BucketAlreadyExists, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-	// Make sure to add Location information here only for bucket
-	w.Header().Set("Location", "/"+bucket)
-	writeSuccessResponse(w, nil)
-}
-
-// PostPolicyBucketHandler - POST policy
-// ----------
-// This implementation of the POST operation handles object creation with a specified
-// signature policy in multipart/form-data
-func (api CloudStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, req *http.Request) {
-	// if body of request is non-nil then check for validity of Content-Length
-	if req.Body != nil {
-		/// if Content-Length is unknown/missing, deny the request
-		if req.ContentLength == -1 {
-			writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
-			return
-		}
-	}
-
-	// Here the parameter is the size of the form data that should
-	// be loaded in memory, the remaining being put in temporary
-	// files
-	reader, e := req.MultipartReader()
-	if e != nil {
-		errorIf(probe.NewError(e), "Unable to initialize multipart reader.", nil)
-		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
-		return
-	}
-
-	fileBody, formValues, err := extractHTTPFormValues(reader)
-	if err != nil {
-		errorIf(err.Trace(), "Unable to parse form values.", nil)
-		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
-		return
-	}
-	bucket := mux.Vars(req)["bucket"]
-	formValues["Bucket"] = bucket
-	object := formValues["Key"]
-	signature, err := initPostPresignedPolicyV4(formValues)
-	if err != nil {
-		errorIf(err.Trace(), "Unable to initialize post policy presigned.", nil)
-		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
-		return
-	}
-	var ok bool
-	if ok, err = signature.DoesPolicySignatureMatch(formValues["X-Amz-Date"]); err != nil {
-		errorIf(err.Trace(), "Unable to verify signature.", nil)
-		writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
-		return
-	}
-	if ok == false {
-		writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
-		return
-	}
-	if err = applyPolicy(formValues); err != nil {
-		errorIf(err.Trace(), "Invalid request, policy doesn't match with the endpoint.", nil)
-		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
-		return
-	}
-	metadata, err := api.Filesystem.CreateObject(bucket, object, "", 0, fileBody, nil)
-	if err != nil {
-		errorIf(err.Trace(), "CreateObject failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.RootPathFull:
-			writeErrorResponse(w, req, RootPathFull, req.URL.Path)
-		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
-		case fs.BadDigest:
-			writeErrorResponse(w, req, BadDigest, req.URL.Path)
-		case fs.SignatureDoesNotMatch:
-			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
-		case fs.IncompleteBody:
-			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
-		case fs.EntityTooLarge:
-			writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
-		case fs.InvalidDigest:
-			writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-	if metadata.MD5 != "" {
-		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
-	}
-	writeSuccessResponse(w, nil)
-}
-
-// PutBucketACLHandler - PUT Bucket ACL
-// ----------
-// This implementation of the PUT operation modifies the bucketACL for authenticated request
-func (api CloudStorageAPI) PutBucketACLHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-		return
-	}
-
-	// read from 'x-amz-acl'
-	aclType := getACLType(req)
-	if aclType == unsupportedACLType {
-		writeErrorResponse(w, req, NotImplemented, req.URL.Path)
-		return
-	}
-	err := api.Filesystem.SetBucketMetadata(bucket, map[string]string{"acl": getACLTypeString(aclType)})
-	if err != nil {
-		errorIf(err.Trace(), "PutBucketACL failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
-		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-	writeSuccessResponse(w, nil)
-}
-
-// GetBucketACLHandler - GET ACL on a Bucket
-// ----------
-// This operation uses acl subresource to the return the ``acl``
-// of a bucket. One must have permission to access the bucket to
-// know its ``acl``. This operation willl return response of 404
-// if bucket not found and 403 for invalid credentials.
-func (api CloudStorageAPI) GetBucketACLHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-		return
-	}
-
-	bucketMetadata, err := api.Filesystem.GetBucketMetadata(bucket)
-	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-	// Generate response
-	response := generateAccessControlPolicyResponse(bucketMetadata.ACL)
-	encodedSuccessResponse := encodeSuccessResponse(response)
-	// Write headers
-	setCommonHeaders(w)
-	// Write success response.
-	writeSuccessResponse(w, encodedSuccessResponse)
-}
-
-// HeadBucketHandler - HEAD Bucket
-// ----------
-// This operation is useful to determine if a bucket exists.
-// The operation returns a 200 OK if the bucket exists and you
-// have permission to access it. Otherwise, the operation might
-// return responses such as 404 Not Found and 403 Forbidden.
-func (api CloudStorageAPI) HeadBucketHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		if api.Filesystem.IsPrivateBucket(bucket) {
-			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-			return
-		}
-	}
-
-	_, err := api.Filesystem.GetBucketMetadata(bucket)
-	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-	writeSuccessResponse(w, nil)
-}
-
-// DeleteBucketHandler - Delete bucket
-func (api CloudStorageAPI) DeleteBucketHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-		return
-	}
-
-	err := api.Filesystem.DeleteBucket(bucket)
-	if err != nil {
-		errorIf(err.Trace(), "DeleteBucket failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-		case fs.BucketNotEmpty:
-			writeErrorResponse(w, req, BucketNotEmpty, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-	writeSuccessNoContent(w)
-}

buildscripts/benchcmp.sh

@@ -0,0 +1,67 @@
+#!/bin/bash          
+
+# usage: ./benchcmp.sh <commit-sha1> <commit-sha2>
+# Exit on any non zero return value on execution of a command. 
+set -e 
+
+# path of benchcmp. 
+benchcmp=${GOPATH}/bin/benchcmp
+
+# function which runs the benchmark comparison. 
+RunBenchCmp () {
+	# Path for storing output of benchmark at commit 1. 
+	commit1Bench=/tmp/minio-$1.bench
+	# Path for storing output of benchmark at commit 2. 
+	commit2Bench=/tmp/minio-$2.bench
+	# switch to commit $1. 
+	git checkout $1
+	# Check if the benchmark results for given commit 1 already exists. 
+	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
+	if [[ ! -f $commit1Bench  ]]
+	then
+
+		echo "Running benchmarks at $1"
+		go test -run=NONE -bench=. | tee $commit1Bench 
+	fi
+	# get back to the commit from which it was started. 
+	git checkout -
+	echo "Checking into commit $2"
+	# switch to commit $2 
+	git checkout $2
+	# Check if the benchmark results for given commit 2 already exists. 
+	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
+	if [[ ! -f $commit2Bench  ]]
+	then
+	# Running benchmarks at $2.
+		echo "Running benchmarks at $2"
+		go test -run=NONE -bench=. | tee $commit2Bench 
+	fi
+	
+	# get back to the commit from which it was started. 
+	git checkout - 
+	# Comparing the benchmarks.
+	echo "Running benchmark comparison between $1 and $2 ..."
+	$benchcmp $commit1Bench $commit2Bench 
+	echo "Done."
+}
+
+# check if 2 commit SHA's of snapshots of code for which benchmp has to be done is provided. 
+if [ ! $# -eq 2 ]
+then
+	# exit if commit SHA's are not provided. 
+	echo $#
+	echo "Need Commit SHA's of 2 snapshots to be supplied to run benchmark comparison."
+	exit 1
+fi
+
+# check if benchcmp exists. 
+if [[ -x "$benchcmp" ]]
+then
+	RunBenchCmp $1 $2
+else
+	# install benchcmp if doesnt't exist. 
+	echo "fetching Benchcmp..."
+	go get -u golang.org/x/tools/cmd/benchcmp
+	echo "Done."
+	RunBenchCmp $1 $2
+fi

buildscripts/build.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+_init() {
+    # Save release LDFLAGS
+    LDFLAGS=$(go run buildscripts/gen-ldflags.go)
+
+    # Extract release tag
+    release_tag=$(echo $LDFLAGS | awk {'print $4'} | cut -f2 -d=)
+
+    # Verify release tag.
+    if [ -z "$release_tag" ]; then
+        echo "Release tag cannot be empty. Please check return value of \`go run buildscripts/gen-ldflags.go\`"
+        exit 1;
+    fi
+
+    # Extract release string.
+    release_str=$(echo $MINIO_RELEASE | tr '[:upper:]' '[:lower:]')
+
+    # Verify release string.
+    if [ -z "$release_str" ]; then
+        echo "Release string cannot be empty. Please set \`MINIO_RELEASE\` env variable."
+        exit 1;
+    fi
+
+    # List of supported architectures
+    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm windows/386 windows/amd64 darwin/amd64 freebsd/amd64'
+
+    ## System binaries
+    CP=`which cp`
+    SHASUM=`which shasum`
+    SED=`which sed`
+}
+
+go_build() {
+    local osarch=$1
+    os=$(echo $osarch | cut -f1 -d'/')
+    arch=$(echo $osarch | cut -f2 -d'/')
+    package=$(go list -f '{{.ImportPath}}')
+    echo -n "-->"
+    printf "%15s:%s\n" "${osarch}" "${package}"
+
+    # Release binary name
+    release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
+    # Release binary downloadable name
+    release_real_bin="$release_str/$os-$arch/$(basename $package)"
+    # Release shasum name
+    release_shasum="$release_str/$os-$arch/$(basename $package).shasum"
+
+    # Go build to build the binary.
+    GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
+
+    # Create copy
+    if [ $os == "windows" ]; then
+        $CP -p $release_bin ${release_real_bin}.exe
+    else
+        $CP -p $release_bin $release_real_bin
+    fi
+
+    # Calculate shasum
+    shasum_str=$(${SHASUM} ${release_bin})
+    echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
+}
+
+main() {
+    # Build releases.
+    echo "Executing $release_str builds for OS: ${SUPPORTED_OSARCH}"
+    echo  "Choose an OS Arch from the below"
+    for osarch in ${SUPPORTED_OSARCH}; do
+        echo ${osarch}
+    done
+
+    read -p "If you want to build for all, Just press Enter: " chosen_osarch
+    if [ "$chosen_osarch" = "" ]; then
+        for each_osarch in ${SUPPORTED_OSARCH}; do
+            go_build ${each_osarch}
+        done
+    else
+        go_build ${chosen_osarch}
+    fi
+
+}
+
+# Run main.
+_init && main

buildscripts/checkdeps.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Minio Cloud Storage, (C) 2015 Minio, Inc.
+# Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,15 +21,21 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GPG_VERSION="1.3"
-    GO_VERSION="1.5.1"
+    GO_VERSION="1.7.1"
     OSX_VERSION="10.8"
-    UNAME=$(uname -sm)
-
-    ## Check all dependencies are present
-    MISSING=""
+    KNAME=$(uname -s)
+    ARCH=$(uname -m)
 }
 
+## FIXME:
+## In OSX, 'readlink -f' option does not exist, hence
+## we have our own readlink -f behaviour here.
+## Once OSX has the option, below function is good enough.
+## 
+## readlink() {
+##     return /bin/readlink -f "$1"
+## }
+##
 readlink() {
     TARGET_FILE=$1
 
@@ -51,174 +57,114 @@ readlink() {
     echo $RESULT
 }
 
-###
-#
-# Takes two arguments
-# arg1: version number in `x.x.x` format
-# arg2: version number in `x.x.x` format
-#
-# example: check_version "$version1" "$version2"
-#
-# returns:
-# 0 - Installed version is equal to required
-# 1 - Installed version is greater than required
-# 2 - Installed version is lesser than required
-# 3 - If args have length zero
-#
-####
-check_version() {
-    ## validate args
-    [[ -z "$1" ]] && return 3
-    [[ -z "$2" ]] && return 3
-
-    if [[ $1 == $2 ]]; then
-        return 0
-    fi
-
-    local IFS=.
-    local i ver1=($1) ver2=($2)
-    # fill empty fields in ver1 with zeros
-    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do
-        ver1[i]=0
-    done
-    for ((i=0; i<${#ver1[@]}; i++)); do
-        if [[ -z ${ver2[i]} ]]; then
-            # fill empty fields in ver2 with zeros
-            ver2[i]=0
-        fi
-        if ((10#${ver1[i]} > 10#${ver2[i]})); then
+## FIXME:
+## In OSX, 'sort -V' option does not exist, hence
+## we have our own version compare function.
+## Once OSX has the option, below function is good enough.
+##
+## check_minimum_version() {
+##     versions=($(echo -e "$1\n$2" | sort -V))
+##     return [ "$1" == "${versions[0]}" ]
+## }
+##
+check_minimum_version() {
+    IFS='.' read -r -a varray1 <<< "$1"
+    IFS='.' read -r -a varray2 <<< "$2"
 
+    for i in "${!varray1[@]}"; do
+        if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then
+            return 0
+        elif [[ ${varray1[i]} -gt ${varray2[i]} ]]; then
             return 1
         fi
-        if ((10#${ver1[i]} < 10#${ver2[i]})); then
-            ## Installed version is lesser than required - Bad condition
-            return 2
-        fi
     done
+
     return 0
 }
 
-check_golang_env() {
-    echo ${GOROOT:?} 2>&1 >/dev/null
-    if [ $? -eq 1 ]; then
-        echo "ERROR"
-        echo "GOROOT environment variable missing, please refer to Go installation document"
-        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
-        exit 1
-    fi
-
-    echo ${GOPATH:?} 2>&1 >/dev/null
-    if [ $? -eq 1 ]; then
-        echo "ERROR"
-        echo "GOPATH environment variable missing, please refer to Go installation document"
-        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
-        exit 1
-    fi
-
-    local go_binary_path=$(which go)
-
-    if [ -z "${go_binary_path}" ] ; then
-        echo "Cannot find go binary in your PATH configuration, please refer to Go installation document"
-        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
-        exit -1
-    fi
-
-    local new_go_binary_path=${go_binary_path}
-    if [ -h "${go_binary_path}" ]; then
-        new_go_binary_path=$(readlink ${go_binary_path})
-    fi
-
-    if [[ !"$(dirname ${new_go_binary_path})" =~ *"${GOROOT%%*(/)}"* ]] ; then
-        echo "The go binary found in your PATH configuration does not belong to the Go installation pointed by your GOROOT environment," \
-            "please refer to Go installation document"
-        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
-        exit -1
-    fi
-}
-
-is_supported_os() {
-    case ${UNAME%% *} in
-        "Linux")
-            os="linux"
-            ;;
-        "Darwin")
-            osx_host_version=$(env sw_vers -productVersion)
-            check_version "${osx_host_version}" "${OSX_VERSION}"
-            [[ $? -ge 2 ]] && die "Minimum OSX version supported is ${OSX_VERSION}"
-            ;;
-        "*")
-            echo "Exiting.. unsupported operating system found"
-            exit 1;
-    esac
-}
-
-is_supported_arch() {
-    local supported
-    case ${UNAME##* } in
-        "x86_64")
-            supported=1
-            ;;
-        "arm"*)
-            supported=1
+assert_is_supported_arch() {
+    case "${ARCH}" in
+        x86_64 | amd64 | aarch64 | arm* )
+            return
             ;;
         *)
-            supported=0
-            ;;
+            echo "ERROR"
+            echo "Arch '${ARCH}' not supported."
+            echo "Supported Arch: [x86_64, amd64, aarch64, arm*]"
+            exit 1
     esac
-    if [ $supported -eq 0 ]; then
-        echo "Invalid arch: ${UNAME} not supported, please use x86_64/amd64"
-        exit 1;
-    fi
 }
 
-check_deps() {
-    check_version "$(env go version 2>/dev/null | sed 's/^.* go\([0-9.]*\).*$/\1/')" "${GO_VERSION}"
-    if [ $? -ge 2 ]; then
-        MISSING="${MISSING} golang(${GO_VERSION})"
+assert_is_supported_os() {
+    case "${KNAME}" in
+        Linux | FreeBSD )
+            return
+            ;;
+        Darwin )
+            osx_host_version=$(env sw_vers -productVersion)
+            if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then
+                echo "ERROR"
+                echo "OSX version '${osx_host_version}' not supported."
+                echo "Minimum supported version: ${OSX_VERSION}"
+                exit 1
+            fi
+            return
+            ;;
+        *)
+            echo "ERROR"
+            echo "OS '${KNAME}' is not supported."
+            echo "Supported OS: [Linux, FreeBSD, Darwin]"
+            exit 1
+    esac
+}
+
+assert_check_golang_env() {
+    if ! which go >/dev/null 2>&1; then
+        echo "ERROR"
+        echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at"
+        echo "https://docs.minio.io/docs/how-to-install-golang"
+        exit 1
     fi
 
-    check_version "$(env git --version 2>/dev/null | sed -e 's/^.* \([0-9.\].*\).*$/\1/' -e 's/^\([0-9.\]*\).*/\1/g')" "${GIT_VERSION}"
-    if [ $? -ge 2 ]; then
-        MISSING="${MISSING} git"
+    installed_go_version=$(go version | sed 's/^.* go\([0-9.]*\).*$/\1/')
+    if ! check_minimum_version "${GO_VERSION}" "${installed_go_version}"; then
+        echo "ERROR"
+        echo "Go version '${installed_go_version}' not supported."
+        echo "Minimum supported version: ${GO_VERSION}"
+        exit 1
     fi
 
-    check_version "$(env gpg --version 2>/dev/null | sed -e 's/^.* \([0-9.\].*\).*$/\1/' -e 's/^\([0-9.\]*\).*/\1/g' | head -1)" "${GPG_VERSION}"
-    if [ $? -ge 2 ]; then
-        MISSING="${MISSING} gpg"
+    if [ -z "${GOPATH}" ]; then
+        echo "ERROR"
+        echo "GOPATH environment variable missing, please refer to Go installation document"
+        echo "https://docs.minio.io/docs/how-to-install-golang"
+        exit 1
+    fi
+
+}
+
+assert_check_deps() {
+    installed_git_version=$(git version | awk '{print $NF}')
+    if ! check_minimum_version "${GIT_VERSION}" "${installed_git_version}"; then
+        echo "ERROR"
+        echo "Git version '${installed_git_version}' not supported."
+        echo "Minimum supported version: ${GIT_VERSION}"
+        exit 1
     fi
 }
 
 main() {
-    echo -n "Check for supported arch.. "
-    is_supported_arch
+    ## Check for supported arch
+    assert_is_supported_arch
 
-    echo -n "Check for supported os.. "
-    is_supported_os
+    ## Check for supported os
+    assert_is_supported_os
 
-    echo -n "Checking if proper environment variables are set.. "
-    check_golang_env
+    ## Check for Go environment
+    assert_check_golang_env
 
-    echo "Done"
-    echo "Using GOPATH=${GOPATH} and GOROOT=${GOROOT}"
-
-    echo -n "Checking dependencies for Minio.. "
-    check_deps
-
-    ## If dependencies are missing, warn the user and abort
-    if [ "x${MISSING}" != "x" ]; then
-        echo "ERROR"
-        echo
-        echo "The following build tools are missing:"
-        echo
-        echo "** ${MISSING} **"
-        echo
-        echo "Please install them "
-        echo "${MISSING}"
-        echo
-        echo "Follow https://github.com/minio/minio/blob/master/INSTALLGO.md for further instructions"
-        exit 1
-    fi
-    echo "Done"
+    ## Check for dependencies
+    assert_check_deps
 }
 
 _init && main "$@"

buildscripts/checkgopath.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+main() {
+    echo "Checking project is in GOPATH:"
+
+    IFS=':' read -r -a paths <<< "$GOPATH"
+    for path in "${paths[@]}"; do
+        minio_path="$path/src/github.com/minio/minio"
+        if [ -d "$minio_path" ]; then
+            if [ "$minio_path" -ef "$PWD" ]; then
+               exit 0
+            fi
+        fi
+    done
+
+    echo "ERROR"
+    echo "Project not found in ${GOPATH}."
+    echo "Follow instructions at https://github.com/minio/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository"
+    exit 1
+}
+
+main

buildscripts/gen-ldflags.go

@@ -28,16 +28,17 @@ import (
 
 func genLDFlags(version string) string {
 	var ldflagsStr string
-	ldflagsStr = "-X main.minioVersion=" + version + " "
-	ldflagsStr = ldflagsStr + "-X main.minioReleaseTag=" + releaseTag(version) + " "
-	ldflagsStr = ldflagsStr + "-X main.minioCommitID=" + commitID() + " "
-	ldflagsStr = ldflagsStr + "-X main.minioShortCommitID=" + commitID()[:12]
+	ldflagsStr = "-X github.com/minio/minio/cmd.Version=" + version
+	ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag(version)
+	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
+	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
+	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
 	return ldflagsStr
 }
 
 // genReleaseTag prints release tag to the console for easy git tagging.
 func releaseTag(version string) string {
-	relPrefix := "UNOFFICIAL"
+	relPrefix := "DEVELOPMENT"
 	if prefix := os.Getenv("MINIO_RELEASE"); prefix != "" {
 		relPrefix = prefix
 	}

buildscripts/go-coverage.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+echo "" > coverage.txt
+
+for d in $(go list ./... | grep -v vendor); do
+    go test -coverprofile=profile.out -covermode=atomic $d
+    if [ -f profile.out ]; then
+        cat profile.out >> coverage.txt
+        rm profile.out
+    fi
+done

cmd/access-key.go

@@ -0,0 +1,91 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+)
+
+// credential container for access and secret keys.
+type credential struct {
+	AccessKeyID     string `json:"accessKey"`
+	SecretAccessKey string `json:"secretKey"`
+}
+
+const (
+	accessKeyMinLen = 5
+	accessKeyMaxLen = 20
+	secretKeyMinLen = 8
+	secretKeyMaxLen = 40
+)
+
+// isValidAccessKey - validate access key for right length.
+func isValidAccessKey(accessKey string) bool {
+	return len(accessKey) >= accessKeyMinLen && len(accessKey) <= accessKeyMaxLen
+}
+
+// isValidSecretKey - validate secret key for right length.
+func isValidSecretKey(secretKey string) bool {
+	return len(secretKey) >= secretKeyMinLen && len(secretKey) <= secretKeyMaxLen
+}
+
+// mustGenAccessKeys - must generate access credentials.
+func mustGenAccessKeys() (creds credential) {
+	creds, err := genAccessKeys()
+	fatalIf(err, "Unable to generate access keys.")
+	return creds
+}
+
+// genAccessKeys - generate access credentials.
+func genAccessKeys() (credential, error) {
+	accessKeyID, err := genAccessKeyID()
+	if err != nil {
+		return credential{}, err
+	}
+	secretAccessKey, err := genSecretAccessKey()
+	if err != nil {
+		return credential{}, err
+	}
+	creds := credential{
+		AccessKeyID:     string(accessKeyID),
+		SecretAccessKey: string(secretAccessKey),
+	}
+	return creds, nil
+}
+
+// genAccessKeyID - generate random alpha numeric value using only uppercase characters
+// takes input as size in integer
+func genAccessKeyID() ([]byte, error) {
+	alpha := make([]byte, accessKeyMaxLen)
+	if _, err := rand.Read(alpha); err != nil {
+		return nil, err
+	}
+	for i := 0; i < accessKeyMaxLen; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return alpha, nil
+}
+
+// genSecretAccessKey - generate random base64 numeric value from a random seed.
+func genSecretAccessKey() ([]byte, error) {
+	rb := make([]byte, secretKeyMaxLen)
+	if _, err := rand.Read(rb); err != nil {
+		return nil, err
+	}
+	return []byte(base64.StdEncoding.EncodeToString(rb))[:secretKeyMaxLen], nil
+}

cmd/api-datatypes.go

@@ -0,0 +1,41 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+)
+
+// ObjectIdentifier carries key name for the object to delete.
+type ObjectIdentifier struct {
+	ObjectName string `xml:"Key"`
+}
+
+// createBucketConfiguration container for bucket configuration request from client.
+// Used for parsing the location from the request body for MakeBucketbucket.
+type createBucketLocationConfiguration struct {
+	XMLName  xml.Name `xml:"CreateBucketConfiguration" json:"-"`
+	Location string   `xml:"LocationConstraint"`
+}
+
+// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
+type DeleteObjectsRequest struct {
+	// Element to enable quiet mode for the request
+	Quiet bool
+	// List of objects to be deleted
+	Objects []ObjectIdentifier `xml:"Object"`
+}

cmd/api-errors.go

@@ -0,0 +1,663 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+)
+
+// APIError structure
+type APIError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+// APIErrorResponse - error response format
+type APIErrorResponse struct {
+	XMLName    xml.Name `xml:"Error" json:"-"`
+	Code       string
+	Message    string
+	Key        string
+	BucketName string
+	Resource   string
+	RequestID  string `xml:"RequestId"`
+	HostID     string `xml:"HostId"`
+}
+
+// APIErrorCode type of error status.
+type APIErrorCode int
+
+// Error codes, non exhaustive list - http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+const (
+	ErrNone APIErrorCode = iota
+	ErrAccessDenied
+	ErrBadDigest
+	ErrEntityTooSmall
+	ErrEntityTooLarge
+	ErrIncompleteBody
+	ErrInternalError
+	ErrInvalidAccessKeyID
+	ErrInvalidBucketName
+	ErrInvalidDigest
+	ErrInvalidRange
+	ErrInvalidMaxKeys
+	ErrInvalidMaxUploads
+	ErrInvalidMaxParts
+	ErrInvalidPartNumberMarker
+	ErrInvalidRequestBody
+	ErrInvalidCopySource
+	ErrInvalidCopyDest
+	ErrInvalidPolicyDocument
+	ErrInvalidObjectState
+	ErrMalformedXML
+	ErrMissingContentLength
+	ErrMissingContentMD5
+	ErrMissingRequestBodyError
+	ErrNoSuchBucket
+	ErrNoSuchBucketPolicy
+	ErrNoSuchKey
+	ErrNoSuchUpload
+	ErrNotImplemented
+	ErrPreconditionFailed
+	ErrRequestTimeTooSkewed
+	ErrSignatureDoesNotMatch
+	ErrMethodNotAllowed
+	ErrInvalidPart
+	ErrInvalidPartOrder
+	ErrAuthorizationHeaderMalformed
+	ErrMalformedPOSTRequest
+	ErrSignatureVersionNotSupported
+	ErrBucketNotEmpty
+	ErrAllAccessDisabled
+	ErrMalformedPolicy
+	ErrMissingFields
+	ErrMissingCredTag
+	ErrCredMalformed
+	ErrInvalidRegion
+	ErrInvalidService
+	ErrInvalidRequestVersion
+	ErrMissingSignTag
+	ErrMissingSignHeadersTag
+	ErrPolicyAlreadyExpired
+	ErrMalformedDate
+	ErrMalformedPresignedDate
+	ErrMalformedCredentialDate
+	ErrMalformedCredentialRegion
+	ErrMalformedExpires
+	ErrNegativeExpires
+	ErrAuthHeaderEmpty
+	ErrExpiredPresignRequest
+	ErrRequestNotReadyYet
+	ErrUnsignedHeaders
+	ErrMissingDateHeader
+	ErrInvalidQuerySignatureAlgo
+	ErrInvalidQueryParams
+	ErrBucketAlreadyOwnedByYou
+	// Add new error codes here.
+
+	// Bucket notification related errors.
+	ErrEventNotification
+	ErrARNNotification
+	ErrRegionNotification
+	ErrOverlappingFilterNotification
+	ErrFilterNameInvalid
+	ErrFilterNamePrefix
+	ErrFilterNameSuffix
+	ErrFilterValueInvalid
+	ErrOverlappingConfigs
+
+	// S3 extended errors.
+	ErrContentSHA256Mismatch
+
+	// Add new extended error codes here.
+
+	// Minio extended errors.
+	ErrReadQuorum
+	ErrWriteQuorum
+	ErrStorageFull
+	ErrObjectExistsAsDirectory
+	ErrPolicyNesting
+	ErrInvalidObjectName
+	ErrServerNotInitialized
+	// Add new extended error codes here.
+	// Please open a https://github.com/minio/minio/issues before adding
+	// new error codes here.
+)
+
+// error code to APIError structure, these fields carry respective
+// descriptions for all the error responses.
+var errorCodeResponse = map[APIErrorCode]APIError{
+	ErrInvalidCopyDest: {
+		Code:           "InvalidRequest",
+		Description:    "This copy request is illegal because it is trying to copy an object to itself.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCopySource: {
+		Code:           "InvalidArgument",
+		Description:    "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRequestBody: {
+		Code:           "InvalidArgument",
+		Description:    "Body shouldn't be set for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxUploads: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-uploads must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxKeys: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxParts: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPartNumberMarker: {
+		Code:           "InvalidArgument",
+		Description:    "Argument partNumberMarker must be an integer.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPolicyDocument: {
+		Code:           "InvalidPolicyDocument",
+		Description:    "The content of the form does not meet the conditions specified in the policy document.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAccessDenied: {
+		Code:           "AccessDenied",
+		Description:    "Access Denied.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrBadDigest: {
+		Code:           "BadDigest",
+		Description:    "The Content-Md5 you specified did not match what we received.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrEntityTooSmall: {
+		Code:           "EntityTooSmall",
+		Description:    "Your proposed upload is smaller than the minimum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrEntityTooLarge: {
+		Code:           "EntityTooLarge",
+		Description:    "Your proposed upload exceeds the maximum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrIncompleteBody: {
+		Code:           "IncompleteBody",
+		Description:    "You did not provide the number of bytes specified by the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInternalError: {
+		Code:           "InternalError",
+		Description:    "We encountered an internal error, please try again.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	ErrInvalidAccessKeyID: {
+		Code:           "InvalidAccessKeyID",
+		Description:    "The access key ID you provided does not exist in our records.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrInvalidBucketName: {
+		Code:           "InvalidBucketName",
+		Description:    "The specified bucket is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidDigest: {
+		Code:           "InvalidDigest",
+		Description:    "The Content-Md5 you specified is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRange: {
+		Code:           "InvalidRange",
+		Description:    "The requested range is not satisfiable",
+		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
+	},
+	ErrMalformedXML: {
+		Code:           "MalformedXML",
+		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingContentLength: {
+		Code:           "MissingContentLength",
+		Description:    "You must provide the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
+	ErrMissingContentMD5: {
+		Code:           "MissingContentMD5",
+		Description:    "Missing required header for this request: Content-Md5.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingRequestBodyError: {
+		Code:           "MissingRequestBodyError",
+		Description:    "Request body is empty.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
+	ErrNoSuchBucket: {
+		Code:           "NoSuchBucket",
+		Description:    "The specified bucket does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchBucketPolicy: {
+		Code:           "NoSuchBucketPolicy",
+		Description:    "The bucket policy does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchKey: {
+		Code:           "NoSuchKey",
+		Description:    "The specified key does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchUpload: {
+		Code:           "NoSuchUpload",
+		Description:    "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNotImplemented: {
+		Code:           "NotImplemented",
+		Description:    "A header you provided implies functionality that is not implemented",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+	ErrPreconditionFailed: {
+		Code:           "PreconditionFailed",
+		Description:    "At least one of the pre-conditions you specified did not hold",
+		HTTPStatusCode: http.StatusPreconditionFailed,
+	},
+	ErrRequestTimeTooSkewed: {
+		Code:           "RequestTimeTooSkewed",
+		Description:    "The difference between the request time and the server's time is too large.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMethodNotAllowed: {
+		Code:           "MethodNotAllowed",
+		Description:    "The specified method is not allowed against this resource.",
+		HTTPStatusCode: http.StatusMethodNotAllowed,
+	},
+	ErrInvalidPart: {
+		Code:           "InvalidPart",
+		Description:    "One or more of the specified parts could not be found.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPartOrder: {
+		Code:           "InvalidPartOrder",
+		Description:    "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidObjectState: {
+		Code:           "InvalidObjectState",
+		Description:    "The operation is not valid for the current state of the object.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAuthorizationHeaderMalformed: {
+		Code:           "AuthorizationHeaderMalformed",
+		Description:    "The authorization header is malformed; the region is wrong; expecting 'us-east-1'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedPOSTRequest: {
+		Code:           "MalformedPOSTRequest",
+		Description:    "The body of your POST request is not well-formed multipart/form-data.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSignatureVersionNotSupported: {
+		Code:           "InvalidRequest",
+		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrAllAccessDisabled: {
+		Code:           "AllAccessDisabled",
+		Description:    "All access to this bucket has been disabled.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMalformedPolicy: {
+		Code:           "MalformedPolicy",
+		Description:    "Policy has invalid resource.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingFields: {
+		Code:           "MissingFields",
+		Description:    "Missing fields in request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingCredTag: {
+		Code:           "InvalidRequest",
+		Description:    "Missing Credential field for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrCredMalformed: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedDate: {
+		Code:           "MalformedDate",
+		Description:    "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedPresignedDate: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
+	// right Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format \"%s\". This date in the credential must be in the format \"yyyyMMdd\".",
+	// Need changes to make sure variable messages can be constructed.
+	ErrMalformedCredentialDate: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format \"%s\". This date in the credential must be in the format \"yyyyMMdd\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
+	// right Description:    "Error parsing the X-Amz-Credential parameter; the region 'us-east-' is wrong; expecting 'us-east-1'".
+	// Need changes to make sure variable messages can be constructed.
+	ErrMalformedCredentialRegion: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; the region is wrong;",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRegion: {
+		Code:           "InvalidRegion",
+		Description:    "Region does not match.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
+	// right Description:   "Error parsing the X-Amz-Credential parameter; incorrect service \"s4\". This endpoint belongs to \"s3\".".
+	// Need changes to make sure variable messages can be constructed.
+	ErrInvalidService: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
+	// Description:   "Error parsing the X-Amz-Credential parameter; incorrect terminal "aws4_reque". This endpoint uses "aws4_request".
+	// Need changes to make sure variable messages can be constructed.
+	ErrInvalidRequestVersion: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; incorrect terminal. This endpoint uses \"aws4_request\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignTag: {
+		Code:           "AccessDenied",
+		Description:    "Signature header missing Signature field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignHeadersTag: {
+		Code:           "InvalidArgument",
+		Description:    "Signature header missing SignedHeaders field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPolicyAlreadyExpired: {
+		Code:           "AccessDenied",
+		Description:    "Invalid according to Policy: Policy expired.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires should be a number",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNegativeExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires must be non-negative",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthHeaderEmpty: {
+		Code:           "InvalidArgument",
+		Description:    "Authorization header is invalid -- one and only one ' ' (space) required.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingDateHeader: {
+		Code:           "AccessDenied",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQuerySignatureAlgo: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrExpiredPresignRequest: {
+		Code:           "AccessDenied",
+		Description:    "Request has expired",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrRequestNotReadyYet: {
+		Code:           "AccessDenied",
+		Description:    "Request is not valid yet",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	// FIXME: Actual XML error response also contains the header which missed in list of signed header parameters.
+	ErrUnsignedHeaders: {
+		Code:           "AccessDenied",
+		Description:    "There were headers present in the request which were not signed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQueryParams: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketAlreadyOwnedByYou: {
+		Code:           "BucketAlreadyOwnedByYou",
+		Description:    "Your previous request to create the named bucket succeeded and you already own it.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+
+	/// Bucket notification related errors.
+	ErrEventNotification: {
+		Code:           "InvalidArgument",
+		Description:    "A specified event is not supported for notifications.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrARNNotification: {
+		Code:           "InvalidArgument",
+		Description:    "A specified destination ARN does not exist or is not well-formed. Verify the destination ARN.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrRegionNotification: {
+		Code:           "InvalidArgument",
+		Description:    "A specified destination is in a different region than the bucket. You must use a destination that resides in the same region as the bucket.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOverlappingFilterNotification: {
+		Code:           "InvalidArgument",
+		Description:    "An object key name filtering rule defined with overlapping prefixes, overlapping suffixes, or overlapping combinations of prefixes and suffixes for the same event types.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrFilterNameInvalid: {
+		Code:           "InvalidArgument",
+		Description:    "filter rule name must be either prefix or suffix",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrFilterNamePrefix: {
+		Code:           "InvalidArgument",
+		Description:    "Cannot specify more than one prefix rule in a filter.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrFilterNameSuffix: {
+		Code:           "InvalidArgument",
+		Description:    "Cannot specify more than one suffix rule in a filter.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrFilterValueInvalid: {
+		Code:           "InvalidArgument",
+		Description:    "Size of filter rule value cannot exceed 1024 bytes in UTF-8 representation",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOverlappingConfigs: {
+		Code:           "InvalidArgument",
+		Description:    "Configurations overlap. Configurations on the same bucket cannot share a common event type.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
+	/// S3 extensions.
+	ErrContentSHA256Mismatch: {
+		Code:           "XAmzContentSHA256Mismatch",
+		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
+	/// Minio extensions.
+	ErrStorageFull: {
+		Code:           "XMinioStorageFull",
+		Description:    "Storage backend has reached its minimum free disk threshold. Please delete few objects to proceed.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	ErrObjectExistsAsDirectory: {
+		Code:           "XMinioObjectExistsAsDirectory",
+		Description:    "Object name already exists as a directory.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrReadQuorum: {
+		Code:           "XMinioReadQuorum",
+		Description:    "Multiple disk failures, unable to reconstruct data.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrWriteQuorum: {
+		Code:           "XMinioWriteQuorum",
+		Description:    "Multiple disks failures, unable to write data.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrPolicyNesting: {
+		Code:           "XMinioPolicyNesting",
+		Description:    "New bucket policy conflicts with an existing policy. Please try again with new prefix.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrInvalidObjectName: {
+		Code:           "XMinioInvalidObjectName",
+		Description:    "Object name contains unsupported characters. Unsupported characters are `^*|\\\"",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrServerNotInitialized: {
+		Code:           "XMinioServerNotInitialized",
+		Description:    "Server not initialized, please try again.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	// Add your error structure here.
+}
+
+// toAPIErrorCode - Converts embedded errors. Convenience
+// function written to handle all cases where we have known types of
+// errors returned by underlying layers.
+func toAPIErrorCode(err error) (apiErr APIErrorCode) {
+	if err == nil {
+		return ErrNone
+	}
+
+	err = errorCause(err)
+	// Verify if the underlying error is signature mismatch.
+	switch err {
+	case errSignatureMismatch:
+		apiErr = ErrSignatureDoesNotMatch
+	case errContentSHA256Mismatch:
+		apiErr = ErrContentSHA256Mismatch
+	}
+
+	if apiErr != ErrNone {
+		// If there was a match in the above switch case.
+		return apiErr
+	}
+
+	switch err.(type) {
+	case StorageFull:
+		apiErr = ErrStorageFull
+	case BadDigest:
+		apiErr = ErrBadDigest
+	case IncompleteBody:
+		apiErr = ErrIncompleteBody
+	case ObjectExistsAsDirectory:
+		apiErr = ErrObjectExistsAsDirectory
+	case PrefixAccessDenied:
+		apiErr = ErrAccessDenied
+	case BucketNameInvalid:
+		apiErr = ErrInvalidBucketName
+	case BucketNotFound:
+		apiErr = ErrNoSuchBucket
+	case BucketNotEmpty:
+		apiErr = ErrBucketNotEmpty
+	case BucketExists:
+		apiErr = ErrBucketAlreadyOwnedByYou
+	case ObjectNotFound:
+		apiErr = ErrNoSuchKey
+	case ObjectNameInvalid:
+		apiErr = ErrInvalidObjectName
+	case InvalidUploadID:
+		apiErr = ErrNoSuchUpload
+	case InvalidPart:
+		apiErr = ErrInvalidPart
+	case InsufficientWriteQuorum:
+		apiErr = ErrWriteQuorum
+	case InsufficientReadQuorum:
+		apiErr = ErrReadQuorum
+	case UnsupportedDelimiter:
+		apiErr = ErrNotImplemented
+	case InvalidMarkerPrefixCombination:
+		apiErr = ErrNotImplemented
+	case InvalidUploadIDKeyCombination:
+		apiErr = ErrNotImplemented
+	case MalformedUploadID:
+		apiErr = ErrNoSuchUpload
+	case PartTooSmall:
+		apiErr = ErrEntityTooSmall
+	case SHA256Mismatch:
+		apiErr = ErrContentSHA256Mismatch
+	case ObjectTooLarge:
+		apiErr = ErrEntityTooLarge
+	case ObjectTooSmall:
+		apiErr = ErrEntityTooSmall
+	default:
+		apiErr = ErrInternalError
+	}
+
+	return apiErr
+}
+
+// getAPIError provides API Error for input API error code.
+func getAPIError(code APIErrorCode) APIError {
+	return errorCodeResponse[code]
+}
+
+// getErrorResponse gets in standard error and resource value and
+// provides a encodable populated response values
+func getAPIErrorResponse(err APIError, resource string) APIErrorResponse {
+	var data = APIErrorResponse{}
+	data.Code = err.Code
+	data.Message = err.Description
+	if resource != "" {
+		data.Resource = resource
+	}
+	// TODO implement this in future
+	data.RequestID = "3L137"
+	data.HostID = "3L137"
+
+	return data
+}

cmd/api-errors_test.go

@@ -0,0 +1,135 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestAPIErrCode(t *testing.T) {
+	testCases := []struct {
+		err     error
+		errCode APIErrorCode
+	}{
+		// Valid cases.
+		{
+			BadDigest{},
+			ErrBadDigest,
+		},
+		{
+			IncompleteBody{},
+			ErrIncompleteBody,
+		},
+		{
+			ObjectExistsAsDirectory{},
+			ErrObjectExistsAsDirectory,
+		},
+		{
+			BucketNameInvalid{},
+			ErrInvalidBucketName,
+		},
+		{
+			BucketExists{},
+			ErrBucketAlreadyOwnedByYou,
+		},
+		{
+			ObjectNotFound{},
+			ErrNoSuchKey,
+		},
+		{
+			ObjectNameInvalid{},
+			ErrInvalidObjectName,
+		},
+		{
+			InvalidUploadID{},
+			ErrNoSuchUpload,
+		},
+		{
+			InvalidPart{},
+			ErrInvalidPart,
+		},
+		{
+			InsufficientReadQuorum{},
+			ErrReadQuorum,
+		},
+		{
+			InsufficientWriteQuorum{},
+			ErrWriteQuorum,
+		},
+		{
+			UnsupportedDelimiter{},
+			ErrNotImplemented,
+		},
+		{
+			InvalidMarkerPrefixCombination{},
+			ErrNotImplemented,
+		},
+		{
+			InvalidUploadIDKeyCombination{},
+			ErrNotImplemented,
+		},
+		{
+			MalformedUploadID{},
+			ErrNoSuchUpload,
+		},
+		{
+			PartTooSmall{},
+			ErrEntityTooSmall,
+		},
+		{
+			BucketNotEmpty{},
+			ErrBucketNotEmpty,
+		},
+		{
+			BucketNotFound{},
+			ErrNoSuchBucket,
+		},
+		{
+			StorageFull{},
+			ErrStorageFull,
+		},
+		{
+			errSignatureMismatch,
+			ErrSignatureDoesNotMatch,
+		},
+		{
+			errContentSHA256Mismatch,
+			ErrContentSHA256Mismatch,
+		}, // End of all valid cases.
+
+		// Case where err is nil.
+		{
+			nil,
+			ErrNone,
+		},
+
+		// Case where err type is unknown.
+		{
+			errors.New("Custom error"),
+			ErrInternalError,
+		},
+	}
+
+	// Validate all the errors with their API error codes.
+	for i, testCase := range testCases {
+		errCode := toAPIErrorCode(testCase.err)
+		if errCode != testCase.errCode {
+			t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)
+		}
+	}
+}

cmd/api-headers.go

@@ -0,0 +1,87 @@
+/*
+ * Minio Cloud Storage, (C) 2015,2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/xml"
+	"net/http"
+	"runtime"
+	"strconv"
+)
+
+// Static alphanumeric table used for generating unique request ids
+var alphaNumericTable = []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+// newRequestID generates and returns request ID string.
+func newRequestID() string {
+	alpha := make([]byte, 16)
+	rand.Read(alpha)
+	for i := 0; i < 16; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return string(alpha)
+}
+
+// Write http common headers
+func setCommonHeaders(w http.ResponseWriter) {
+	// Set unique request ID for each reply.
+	w.Header().Set("X-Amz-Request-Id", newRequestID())
+	w.Header().Set("Server", ("Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"))
+	w.Header().Set("Accept-Ranges", "bytes")
+}
+
+// Encodes the response headers into XML format.
+func encodeResponse(response interface{}) []byte {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	e.Encode(response)
+	return bytesBuffer.Bytes()
+}
+
+// Write object header
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
+	// set common headers
+	setCommonHeaders(w)
+
+	// Set content length.
+	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
+
+	// Set last modified time.
+	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
+	w.Header().Set("Last-Modified", lastModified)
+
+	// Set Etag if available.
+	if objInfo.MD5Sum != "" {
+		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+	}
+
+	// Set all other user defined metadata.
+	for k, v := range objInfo.UserDefined {
+		w.Header().Set(k, v)
+	}
+
+	// for providing ranged content
+	if contentRange != nil && contentRange.offsetBegin > -1 {
+		// Override content-length
+		w.Header().Set("Content-Length", strconv.FormatInt(contentRange.getLength(), 10))
+		w.Header().Set("Content-Range", contentRange.String())
+		w.WriteHeader(http.StatusPartialContent)
+	}
+}

cmd/api-headers_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,28 +14,27 @@
  * limitations under the License.
  */
 
-package user_test
+package cmd
 
 import (
 	"testing"
-
-	"github.com/minio/minio/pkg/user"
-
-	. "gopkg.in/check.v1"
 )
 
-func Test(t *testing.T) { TestingT(t) }
+func TestNewRequestID(t *testing.T) {
+	// Ensure that it returns an alphanumeric result of length 16.
+	var id = newRequestID()
 
-type MySuite struct{}
+	if len(id) != 16 {
+		t.Fail()
+	}
 
-var _ = Suite(&MySuite{})
+	var e rune
+	for _, char := range id {
+		e = rune(char)
 
-func (s *MySuite) TestUser(c *C) {
-	_, err := user.Current()
-	c.Assert(err, IsNil)
-}
-
-func (s *MySuite) TestHomeDir(c *C) {
-	_, err := user.HomeDir()
-	c.Assert(err, IsNil)
+		// Ensure that it is alphanumeric, in this case, between 0-9 and A-Z.
+		if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) {
+			t.Fail()
+		}
+	}
 }

cmd/api-resources.go

@@ -0,0 +1,98 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/url"
+	"strconv"
+)
+
+// Parse bucket url queries
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse bucket url queries for ListObjects V2.
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
+	prefix = values.Get("prefix")
+	token = values.Get("continuation-token")
+	startAfter = values.Get("start-after")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
+	fetchOwner = values.Get("fetch-owner") == "true"
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse bucket url queries for ?uploads
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+	prefix = values.Get("prefix")
+	keyMarker = values.Get("key-marker")
+	uploadIDMarker = values.Get("upload-id-marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-uploads") != "" {
+		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
+	} else {
+		maxUploads = maxUploadsList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse object url queries
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
+	uploadID = values.Get("uploadId")
+	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+	if values.Get("max-parts") != "" {
+		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+	} else {
+		maxParts = maxPartsList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse listen bucket notification resources.
+func getListenBucketNotificationResources(values url.Values) (prefix []string, suffix []string, events []string) {
+	prefix = values["prefix"]
+	suffix = values["suffix"]
+	events = values["events"]
+	return prefix, suffix, events
+}
+
+// Validates filter values
+func validateFilterValues(values []string) (err APIErrorCode) {
+	for _, value := range values {
+		if !IsValidObjectPrefix(value) {
+			return ErrFilterValueInvalid
+		}
+	}
+	return ErrNone
+}

cmd/api-resources_test.go

@@ -0,0 +1,226 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+)
+
+// Test list objects resources V2.
+func TestListObjectsV2Resources(t *testing.T) {
+	testCases := []struct {
+		values                               url.Values
+		prefix, token, startAfter, delimiter string
+		fetchOwner                           bool
+		maxKeys                              int
+		encodingType                         string
+	}{
+		{
+			values: url.Values{
+				"prefix":             []string{"photos/"},
+				"continuation-token": []string{"token"},
+				"start-after":        []string{"start-after"},
+				"delimiter":          []string{"/"},
+				"fetch-owner":        []string{"true"},
+				"max-keys":           []string{"100"},
+				"encoding-type":      []string{"gzip"},
+			},
+			prefix:       "photos/",
+			token:        "token",
+			startAfter:   "start-after",
+			delimiter:    "/",
+			fetchOwner:   true,
+			maxKeys:      100,
+			encodingType: "gzip",
+		},
+		{
+			values: url.Values{
+				"prefix":             []string{"photos/"},
+				"continuation-token": []string{"token"},
+				"start-after":        []string{"start-after"},
+				"delimiter":          []string{"/"},
+				"fetch-owner":        []string{"true"},
+				"encoding-type":      []string{"gzip"},
+			},
+			prefix:       "photos/",
+			token:        "token",
+			startAfter:   "start-after",
+			delimiter:    "/",
+			fetchOwner:   true,
+			maxKeys:      1000,
+			encodingType: "gzip",
+		},
+	}
+
+	for i, testCase := range testCases {
+		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType := getListObjectsV2Args(testCase.values)
+		if prefix != testCase.prefix {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
+		}
+		if token != testCase.token {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.token, token)
+		}
+		if startAfter != testCase.startAfter {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.startAfter, startAfter)
+		}
+		if delimiter != testCase.delimiter {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.delimiter, delimiter)
+		}
+		if fetchOwner != testCase.fetchOwner {
+			t.Errorf("Test %d: Expected %t, got %t", i+1, testCase.fetchOwner, fetchOwner)
+		}
+		if maxKeys != testCase.maxKeys {
+			t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.maxKeys, maxKeys)
+		}
+		if encodingType != testCase.encodingType {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.encodingType, encodingType)
+		}
+	}
+}
+
+// Test list objects resources V1.
+func TestListObjectsV1Resources(t *testing.T) {
+	testCases := []struct {
+		values                    url.Values
+		prefix, marker, delimiter string
+		maxKeys                   int
+		encodingType              string
+	}{
+		{
+			values: url.Values{
+				"prefix":        []string{"photos/"},
+				"marker":        []string{"test"},
+				"delimiter":     []string{"/"},
+				"max-keys":      []string{"100"},
+				"encoding-type": []string{"gzip"},
+			},
+			prefix:       "photos/",
+			marker:       "test",
+			delimiter:    "/",
+			maxKeys:      100,
+			encodingType: "gzip",
+		},
+		{
+			values: url.Values{
+				"prefix":        []string{"photos/"},
+				"marker":        []string{"test"},
+				"delimiter":     []string{"/"},
+				"encoding-type": []string{"gzip"},
+			},
+			prefix:       "photos/",
+			marker:       "test",
+			delimiter:    "/",
+			maxKeys:      1000,
+			encodingType: "gzip",
+		},
+	}
+
+	for i, testCase := range testCases {
+		prefix, marker, delimiter, maxKeys, encodingType := getListObjectsV1Args(testCase.values)
+		if prefix != testCase.prefix {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
+		}
+		if marker != testCase.marker {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.marker, marker)
+		}
+		if delimiter != testCase.delimiter {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.delimiter, delimiter)
+		}
+		if maxKeys != testCase.maxKeys {
+			t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.maxKeys, maxKeys)
+		}
+		if encodingType != testCase.encodingType {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.encodingType, encodingType)
+		}
+	}
+}
+
+// Validates extracting information for object resources.
+func TestGetObjectsResources(t *testing.T) {
+	testCases := []struct {
+		values                     url.Values
+		uploadID                   string
+		partNumberMarker, maxParts int
+		encodingType               string
+	}{
+		{
+			values: url.Values{
+				"uploadId":           []string{"11123-11312312311231-12313"},
+				"part-number-marker": []string{"1"},
+				"max-parts":          []string{"1000"},
+				"encoding-type":      []string{"gzip"},
+			},
+			uploadID:         "11123-11312312311231-12313",
+			partNumberMarker: 1,
+			maxParts:         1000,
+			encodingType:     "gzip",
+		},
+	}
+
+	for i, testCase := range testCases {
+		uploadID, partNumberMarker, maxParts, encodingType := getObjectResources(testCase.values)
+		if uploadID != testCase.uploadID {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.uploadID, uploadID)
+		}
+		if partNumberMarker != testCase.partNumberMarker {
+			t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.partNumberMarker, partNumberMarker)
+		}
+		if maxParts != testCase.maxParts {
+			t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.maxParts, maxParts)
+		}
+		if encodingType != testCase.encodingType {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.encodingType, encodingType)
+		}
+	}
+}
+
+// Validates if filter values are correct
+func TestValidateFilterValues(t *testing.T) {
+	testCases := []struct {
+		values        []string
+		expectedError APIErrorCode
+	}{
+		{
+			values:        []string{""},
+			expectedError: ErrNone,
+		},
+		{
+			values:        []string{"", "prefix"},
+			expectedError: ErrNone,
+		},
+		{
+			values:        []string{strings.Repeat("a", 1025)},
+			expectedError: ErrFilterValueInvalid,
+		},
+		{
+			values:        []string{"a\\b"},
+			expectedError: ErrFilterValueInvalid,
+		},
+		{
+			values:        []string{string([]byte{0xff, 0xfe, 0xfd})},
+			expectedError: ErrFilterValueInvalid,
+		},
+	}
+
+	for i, testCase := range testCases {
+		if actualError := validateFilterValues(testCase.values); actualError != testCase.expectedError {
+			t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.expectedError, actualError)
+		}
+	}
+}

cmd/api-response-multipart.go

@@ -0,0 +1,56 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "net/http"
+
+// Represents additional fields necessary for ErrPartTooSmall S3 error.
+type completeMultipartAPIError struct {
+	// Proposed size represents uploaded size of the part.
+	ProposedSize int64
+	// Minimum size allowed epresents the minimum size allowed per
+	// part. Defaults to 5MB.
+	MinSizeAllowed int64
+	// Part number of the part which is incorrect.
+	PartNumber int
+	// ETag of the part which is incorrect.
+	PartETag string
+	// Other default XML error responses.
+	APIErrorResponse
+}
+
+// writeErrorResponsePartTooSmall - function is used specifically to
+// construct a proper error response during CompleteMultipartUpload
+// when one of the parts is < 5MB.
+// The requirement comes due to the fact that generic ErrorResponse
+// XML doesn't carry the additional fields required to send this
+// error. So we construct a new type which lies well within the scope
+// of this function.
+func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
+
+	apiError := getAPIError(toAPIErrorCode(err))
+	// Generate complete multipart error response.
+	errorResponse := getAPIErrorResponse(apiError, r.URL.Path)
+	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
+	encodedErrorResponse := encodeResponse(cmpErrResp)
+
+	// respond with 400 bad request.
+	w.WriteHeader(apiError.HTTPStatusCode)
+	// Write error body.
+	w.Write(encodedErrorResponse)
+	w.(http.Flusher).Flush()
+}

cmd/api-response.go

@@ -0,0 +1,515 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+	"path"
+	"time"
+)
+
+const (
+	timeFormatAMZ     = "2006-01-02T15:04:05Z"     // Reply date format
+	timeFormatAMZLong = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
+	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse.
+	maxUploadsList    = 1000                       // Limit number of uploads in a listUploadsResponse.
+	maxPartsList      = 1000                       // Limit number of parts in a listPartsResponse.
+)
+
+// LocationResponse - format for location response.
+type LocationResponse struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
+	Location string   `xml:",chardata"`
+}
+
+// ListObjectsResponse - format for list objects response.
+type ListObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+
+	Name   string
+	Prefix string
+	Marker string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	NextMarker string `xml:"NextMarker,omitempty"`
+
+	MaxKeys   int
+	Delimiter string
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+
+	Contents       []Object
+	CommonPrefixes []CommonPrefix
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string `xml:"EncodingType,omitempty"`
+}
+
+// ListObjectsV2Response - format for list objects response.
+type ListObjectsV2Response struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+
+	Name       string
+	Prefix     string
+	StartAfter string `xml:"StartAfter,omitempty"`
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	ContinuationToken     string `xml:"ContinuationToken,omitempty"`
+	NextContinuationToken string `xml:"NextContinuationToken,omitempty"`
+
+	KeyCount  int
+	MaxKeys   int
+	Delimiter string
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+
+	Contents       []Object
+	CommonPrefixes []CommonPrefix
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string `xml:"EncodingType,omitempty"`
+}
+
+// Part container for part metadata.
+type Part struct {
+	PartNumber   int
+	LastModified string
+	ETag         string
+	Size         int64
+}
+
+// ListPartsResponse - format for list parts response.
+type ListPartsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+
+	Initiator Initiator
+	Owner     Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	// List of parts.
+	Parts []Part `xml:"Part"`
+}
+
+// ListMultipartUploadsResponse - format for list multipart uploads response.
+type ListMultipartUploadsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
+
+	Bucket             string
+	KeyMarker          string
+	UploadIDMarker     string `xml:"UploadIdMarker"`
+	NextKeyMarker      string
+	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
+	Delimiter          string
+	Prefix             string
+	EncodingType       string `xml:"EncodingType,omitempty"`
+	MaxUploads         int
+	IsTruncated        bool
+
+	// List of pending uploads.
+	Uploads []Upload `xml:"Upload"`
+
+	// Delimed common prefixes.
+	CommonPrefixes []CommonPrefix
+}
+
+// ListBucketsResponse - format for list buckets response
+type ListBucketsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
+
+	Owner Owner
+
+	// Container for one or more buckets.
+	Buckets struct {
+		Buckets []Bucket `xml:"Bucket"`
+	} // Buckets are nested
+}
+
+// Upload container for in progress multipart upload
+type Upload struct {
+	Key          string
+	UploadID     string `xml:"UploadId"`
+	Initiator    Initiator
+	Owner        Owner
+	StorageClass string
+	Initiated    string
+}
+
+// CommonPrefix container for prefix response in ListObjectsResponse
+type CommonPrefix struct {
+	Prefix string
+}
+
+// Bucket container for bucket metadata
+type Bucket struct {
+	Name         string
+	CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
+}
+
+// Object container for object metadata
+type Object struct {
+	Key          string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+	ETag         string
+	Size         int64
+
+	// Owner of the object.
+	Owner Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+}
+
+// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
+type CopyObjectResponse struct {
+	XMLName      xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
+	LastModified string   // time string of format "2006-01-02T15:04:05.000Z"
+	ETag         string   // md5sum of the copied object.
+}
+
+// Initiator inherit from Owner struct, fields are same
+type Initiator Owner
+
+// Owner - bucket owner/principal
+type Owner struct {
+	ID          string
+	DisplayName string
+}
+
+// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
+type InitiateMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+}
+
+// CompleteMultipartUploadResponse container for completed multipart upload response
+type CompleteMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
+
+	Location string
+	Bucket   string
+	Key      string
+	ETag     string
+}
+
+// DeleteError structure.
+type DeleteError struct {
+	Code    string
+	Message string
+	Key     string
+}
+
+// DeleteObjectsResponse container for multiple object deletes.
+type DeleteObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`
+
+	// Collection of all deleted objects
+	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`
+
+	// Collection of errors deleting certain objects.
+	Errors []DeleteError `xml:"Error,omitempty"`
+}
+
+// getLocation get URL location.
+func getLocation(r *http.Request) string {
+	return path.Clean(r.URL.Path) // Clean any trailing slashes.
+}
+
+// getObjectLocation gets the relative URL for an object
+func getObjectLocation(bucketName string, key string) string {
+	return "/" + bucketName + "/" + key
+}
+
+// generates ListBucketsResponse from array of BucketInfo which can be
+// serialized to match XML and JSON API spec output.
+func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
+	var listbuckets []Bucket
+	var data = ListBucketsResponse{}
+	var owner = Owner{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, bucket := range buckets {
+		var listbucket = Bucket{}
+		listbucket.Name = bucket.Name
+		listbucket.CreationDate = bucket.Created.Format(timeFormatAMZLong)
+		listbuckets = append(listbuckets, listbucket)
+	}
+
+	data.Owner = owner
+	data.Buckets.Buckets = listbuckets
+
+	return data
+}
+
+// generates an ListObjectsV1 response for the said bucket with other enumerated options.
+func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
+	var contents []Object
+	var prefixes []CommonPrefix
+	var owner = Owner{}
+	var data = ListObjectsResponse{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, object := range resp.Objects {
+		var content = Object{}
+		if object.Name == "" {
+			continue
+		}
+		content.Key = object.Name
+		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
+		}
+		content.Size = object.Size
+		content.StorageClass = "STANDARD"
+		content.Owner = owner
+		contents = append(contents, content)
+	}
+	// TODO - support EncodingType in xml decoding
+	data.Name = bucket
+	data.Contents = contents
+
+	data.Prefix = prefix
+	data.Marker = marker
+	data.Delimiter = delimiter
+	data.MaxKeys = maxKeys
+
+	data.NextMarker = resp.NextMarker
+	data.IsTruncated = resp.IsTruncated
+	for _, prefix := range resp.Prefixes {
+		var prefixItem = CommonPrefix{}
+		prefixItem.Prefix = prefix
+		prefixes = append(prefixes, prefixItem)
+	}
+	data.CommonPrefixes = prefixes
+	return data
+}
+
+// generates an ListObjectsV2 response for the said bucket with other enumerated options.
+func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, fetchOwner bool, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
+	var contents []Object
+	var prefixes []CommonPrefix
+	var owner = Owner{}
+	var data = ListObjectsV2Response{}
+
+	if fetchOwner {
+		owner.ID = "minio"
+		owner.DisplayName = "minio"
+	}
+
+	for _, object := range resp.Objects {
+		var content = Object{}
+		if object.Name == "" {
+			continue
+		}
+		content.Key = object.Name
+		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
+		}
+		content.Size = object.Size
+		content.StorageClass = "STANDARD"
+		content.Owner = owner
+		contents = append(contents, content)
+	}
+	// TODO - support EncodingType in xml decoding
+	data.Name = bucket
+	data.Contents = contents
+
+	data.StartAfter = startAfter
+	data.Delimiter = delimiter
+	data.Prefix = prefix
+	data.MaxKeys = maxKeys
+	data.ContinuationToken = token
+	data.NextContinuationToken = resp.NextMarker
+	data.IsTruncated = resp.IsTruncated
+	for _, prefix := range resp.Prefixes {
+		var prefixItem = CommonPrefix{}
+		prefixItem.Prefix = prefix
+		prefixes = append(prefixes, prefixItem)
+	}
+	data.CommonPrefixes = prefixes
+	data.KeyCount = len(data.Contents) + len(data.CommonPrefixes)
+	return data
+}
+
+// generates CopyObjectResponse from etag and lastModified time.
+func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
+	return CopyObjectResponse{
+		ETag:         "\"" + etag + "\"",
+		LastModified: lastModified.UTC().Format(timeFormatAMZLong),
+	}
+}
+
+// generates InitiateMultipartUploadResponse for given bucket, key and uploadID.
+func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse {
+	return InitiateMultipartUploadResponse{
+		Bucket:   bucket,
+		Key:      key,
+		UploadID: uploadID,
+	}
+}
+
+// generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
+func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
+	return CompleteMultipartUploadResponse{
+		Location: location,
+		Bucket:   bucket,
+		Key:      key,
+		ETag:     etag,
+	}
+}
+
+// generates ListPartsResponse from ListPartsInfo.
+func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
+	// TODO - support EncodingType in xml decoding
+	listPartsResponse := ListPartsResponse{}
+	listPartsResponse.Bucket = partsInfo.Bucket
+	listPartsResponse.Key = partsInfo.Object
+	listPartsResponse.UploadID = partsInfo.UploadID
+	listPartsResponse.StorageClass = "STANDARD"
+	listPartsResponse.Initiator.ID = "minio"
+	listPartsResponse.Initiator.DisplayName = "minio"
+	listPartsResponse.Owner.ID = "minio"
+	listPartsResponse.Owner.DisplayName = "minio"
+
+	listPartsResponse.MaxParts = partsInfo.MaxParts
+	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
+	listPartsResponse.IsTruncated = partsInfo.IsTruncated
+	listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
+
+	listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
+	for index, part := range partsInfo.Parts {
+		newPart := Part{}
+		newPart.PartNumber = part.PartNumber
+		newPart.ETag = "\"" + part.ETag + "\""
+		newPart.Size = part.Size
+		newPart.LastModified = part.LastModified.UTC().Format(timeFormatAMZLong)
+		listPartsResponse.Parts[index] = newPart
+	}
+	return listPartsResponse
+}
+
+// generates ListMultipartUploadsResponse for given bucket and ListMultipartsInfo.
+func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse {
+	listMultipartUploadsResponse := ListMultipartUploadsResponse{}
+	listMultipartUploadsResponse.Bucket = bucket
+	listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter
+	listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated
+	listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType
+	listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix
+	listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker
+	listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker
+	listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads
+	listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker
+	listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker
+	listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes))
+	for index, commonPrefix := range multipartsInfo.CommonPrefixes {
+		listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{
+			Prefix: commonPrefix,
+		}
+	}
+	listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads))
+	for index, upload := range multipartsInfo.Uploads {
+		newUpload := Upload{}
+		newUpload.UploadID = upload.UploadID
+		newUpload.Key = upload.Object
+		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong)
+		listMultipartUploadsResponse.Uploads[index] = newUpload
+	}
+	return listMultipartUploadsResponse
+}
+
+// generate multi objects delete response.
+func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier, errs []DeleteError) DeleteObjectsResponse {
+	deleteResp := DeleteObjectsResponse{}
+	if !quiet {
+		deleteResp.DeletedObjects = deletedObjects
+	}
+	deleteResp.Errors = errs
+	return deleteResp
+}
+
+// writeSuccessResponse write success headers and response if any.
+func writeSuccessResponse(w http.ResponseWriter, response []byte) {
+	setCommonHeaders(w)
+	if response == nil {
+		w.WriteHeader(http.StatusOK)
+		return
+	}
+	w.Write(response)
+	w.(http.Flusher).Flush()
+}
+
+// writeSuccessNoContent write success headers with http status 204
+func writeSuccessNoContent(w http.ResponseWriter) {
+	setCommonHeaders(w)
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// writeErrorRespone write error headers
+func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorCode APIErrorCode, resource string) {
+	apiError := getAPIError(errorCode)
+	// set common headers
+	setCommonHeaders(w)
+	// write Header
+	w.WriteHeader(apiError.HTTPStatusCode)
+	writeErrorResponseNoHeader(w, req, errorCode, resource)
+}
+
+func writeErrorResponseNoHeader(w http.ResponseWriter, req *http.Request, errorCode APIErrorCode, resource string) {
+	apiError := getAPIError(errorCode)
+	// Generate error response.
+	errorResponse := getAPIErrorResponse(apiError, resource)
+	encodedErrorResponse := encodeResponse(errorResponse)
+	// HEAD should have no body, do not attempt to write to it
+	if req.Method != "HEAD" {
+		// write error body
+		w.Write(encodedErrorResponse)
+		w.(http.Flusher).Flush()
+	}
+}

cmd/api-router.go

@@ -0,0 +1,99 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import router "github.com/gorilla/mux"
+
+// objectAPIHandler implements and provides http handlers for S3 API.
+type objectAPIHandlers struct {
+	ObjectAPI func() ObjectLayer
+}
+
+// registerAPIRouter - registers S3 compatible APIs.
+func registerAPIRouter(mux *router.Router) {
+	// Initialize API.
+	api := objectAPIHandlers{
+		ObjectAPI: newObjectLayerFn,
+	}
+
+	// API Router
+	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
+
+	// Bucket router
+	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+
+	/// Object operations
+
+	// HeadObject
+	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
+	// PutObjectPart
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	// ListObjectPxarts
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
+	// CompleteMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// NewMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
+	// AbortMultipartUpload
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// GetObject
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
+	// CopyObject
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
+	// PutObject
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
+	// DeleteObject
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
+
+	/// Bucket operations
+
+	// GetBucketLocation
+	bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
+	// GetBucketPolicy
+	bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
+	// GetBucketNotification
+	bucket.Methods("GET").HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "")
+	// ListenBucketNotification
+	bucket.Methods("GET").HandlerFunc(api.ListenBucketNotificationHandler).Queries("events", "{events:.*}")
+	// ListMultipartUploads
+	bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
+	// ListObjectsV2
+	bucket.Methods("GET").HandlerFunc(api.ListObjectsV2Handler).Queries("list-type", "2")
+	// ListObjectsV1 (Legacy)
+	bucket.Methods("GET").HandlerFunc(api.ListObjectsV1Handler)
+	// PutBucketPolicy
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
+	// PutBucketNotification
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "")
+	// PutBucket
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
+	// HeadBucket
+	bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
+	// PostPolicy
+	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
+	// DeleteMultipleObjects
+	bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
+	// DeleteBucketPolicy
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
+	// DeleteBucket
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
+
+	/// Root operation
+
+	// ListBuckets
+	apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
+}

cmd/auth-handler.go

@@ -0,0 +1,235 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// Verify if the request http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD"
+func isRequestUnsignedPayload(r *http.Request) bool {
+	return r.Header.Get("x-amz-content-sha256") == unsignedPayload
+}
+
+// Verify if request has JWT.
+func isRequestJWT(r *http.Request) bool {
+	return strings.HasPrefix(r.Header.Get("Authorization"), jwtAlgorithm)
+}
+
+// Verify if request has AWS Signature Version '4'.
+func isRequestSignatureV4(r *http.Request) bool {
+	return strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm)
+}
+
+// Verify if request has AWS Signature Version '2'.
+func isRequestSignatureV2(r *http.Request) bool {
+	return (!strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm) &&
+		strings.HasPrefix(r.Header.Get("Authorization"), signV2Algorithm))
+}
+
+// Verify if request has AWS PreSign Version '4'.
+func isRequestPresignedSignatureV4(r *http.Request) bool {
+	_, ok := r.URL.Query()["X-Amz-Credential"]
+	return ok
+}
+
+// Verify request has AWS PreSign Version '2'.
+func isRequestPresignedSignatureV2(r *http.Request) bool {
+	_, ok := r.URL.Query()["AWSAccessKeyId"]
+	return ok
+}
+
+// Verify if request has AWS Post policy Signature Version '4'.
+func isRequestPostPolicySignatureV4(r *http.Request) bool {
+	return strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") && r.Method == "POST"
+}
+
+// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
+func isRequestSignStreamingV4(r *http.Request) bool {
+	return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 && r.Method == "PUT"
+}
+
+// Authorization type.
+type authType int
+
+// List of all supported auth types.
+const (
+	authTypeUnknown authType = iota
+	authTypeAnonymous
+	authTypePresigned
+	authTypePresignedV2
+	authTypePostPolicy
+	authTypeStreamingSigned
+	authTypeSigned
+	authTypeSignedV2
+	authTypeJWT
+)
+
+// Get request authentication type.
+func getRequestAuthType(r *http.Request) authType {
+	if isRequestSignatureV2(r) {
+		return authTypeSignedV2
+	} else if isRequestPresignedSignatureV2(r) {
+		return authTypePresignedV2
+	} else if isRequestSignStreamingV4(r) {
+		return authTypeStreamingSigned
+	} else if isRequestSignatureV4(r) {
+		return authTypeSigned
+	} else if isRequestPresignedSignatureV4(r) {
+		return authTypePresigned
+	} else if isRequestJWT(r) {
+		return authTypeJWT
+	} else if isRequestPostPolicySignatureV4(r) {
+		return authTypePostPolicy
+	} else if _, ok := r.Header["Authorization"]; !ok {
+		return authTypeAnonymous
+	}
+	return authTypeUnknown
+}
+
+func checkRequestAuthType(r *http.Request, bucket, policyAction, region string) APIErrorCode {
+	reqAuthType := getRequestAuthType(r)
+
+	switch reqAuthType {
+	case authTypePresignedV2, authTypeSignedV2:
+		// Signature V2 validation.
+		s3Error := isReqAuthenticatedV2(r)
+		if s3Error != ErrNone {
+			errorIf(errSignatureMismatch, dumpRequest(r))
+		}
+		return s3Error
+	case authTypeSigned, authTypePresigned:
+		s3Error := isReqAuthenticated(r, region)
+		if s3Error != ErrNone {
+			errorIf(errSignatureMismatch, dumpRequest(r))
+		}
+		return s3Error
+	}
+
+	if reqAuthType == authTypeAnonymous && policyAction != "" {
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		return enforceBucketPolicy(bucket, policyAction, r.URL)
+	}
+
+	// By default return ErrAccessDenied
+	return ErrAccessDenied
+}
+
+// Verify if request has valid AWS Signature Version '2'.
+func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) {
+	if isRequestSignatureV2(r) {
+		return doesSignV2Match(r)
+	}
+	return doesPresignV2SignatureMatch(r)
+}
+
+func reqSignatureV4Verify(r *http.Request) (s3Error APIErrorCode) {
+	sha256sum := r.Header.Get("X-Amz-Content-Sha256")
+	// Skips calculating sha256 on the payload on server,
+	// if client requested for it.
+	if skipContentSha256Cksum(r) {
+		sha256sum = unsignedPayload
+	}
+	if isRequestSignatureV4(r) {
+		return doesSignatureMatch(sha256sum, r, serverConfig.GetRegion())
+	} else if isRequestPresignedSignatureV4(r) {
+		return doesPresignedSignatureMatch(sha256sum, r, serverConfig.GetRegion())
+	}
+	return ErrAccessDenied
+}
+
+// Verify if request has valid AWS Signature Version '4'.
+func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
+	if r == nil {
+		return ErrInternalError
+	}
+	payload, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		errorIf(err, "Unable to read request body for signature verification")
+		return ErrInternalError
+	}
+	// Verify Content-Md5, if payload is set.
+	if r.Header.Get("Content-Md5") != "" {
+		if r.Header.Get("Content-Md5") != getMD5HashBase64(payload) {
+			return ErrBadDigest
+		}
+	}
+	// Populate back the payload.
+	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+	// Skips calculating sha256 on the payload on server, if client requested for it.
+	var sha256sum string
+	if skipContentSha256Cksum(r) {
+		sha256sum = unsignedPayload
+	} else {
+		sha256sum = getSHA256Hash(payload)
+	}
+	if isRequestSignatureV4(r) {
+		return doesSignatureMatch(sha256sum, r, region)
+	} else if isRequestPresignedSignatureV4(r) {
+		return doesPresignedSignatureMatch(sha256sum, r, region)
+	}
+	return ErrAccessDenied
+}
+
+// authHandler - handles all the incoming authorization headers and validates them if possible.
+type authHandler struct {
+	handler http.Handler
+}
+
+// setAuthHandler to validate authorization header for the incoming request.
+func setAuthHandler(h http.Handler) http.Handler {
+	return authHandler{h}
+}
+
+// List of all support S3 auth types.
+var supportedS3AuthTypes = map[authType]struct{}{
+	authTypeAnonymous:       {},
+	authTypePresigned:       {},
+	authTypePresignedV2:     {},
+	authTypeSigned:          {},
+	authTypeSignedV2:        {},
+	authTypePostPolicy:      {},
+	authTypeStreamingSigned: {},
+}
+
+// Validate if the authType is valid and supported.
+func isSupportedS3AuthType(aType authType) bool {
+	_, ok := supportedS3AuthTypes[aType]
+	return ok
+}
+
+// handler for validating incoming authorization headers.
+func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	aType := getRequestAuthType(r)
+	if isSupportedS3AuthType(aType) {
+		// Let top level caller validate for anonymous and known signed requests.
+		a.handler.ServeHTTP(w, r)
+		return
+	} else if aType == authTypeJWT {
+		// Validate Authorization header if its valid for JWT request.
+		if !isJWTReqAuthenticated(r) {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		a.handler.ServeHTTP(w, r)
+		return
+	}
+	writeErrorResponse(w, r, ErrSignatureVersionNotSupported, r.URL.Path)
+}

cmd/auth-handler_test.go

@@ -0,0 +1,344 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"net/url"
+	"testing"
+)
+
+// Test get request auth type.
+func TestGetRequestAuthType(t *testing.T) {
+	type testCase struct {
+		req   *http.Request
+		authT authType
+	}
+	testCases := []testCase{
+		// Test case - 1
+		// Check for generic signature v4 header.
+		{
+			req: &http.Request{
+				URL: &url.URL{
+					Host:   "localhost:9000",
+					Scheme: "http",
+					Path:   "/",
+				},
+				Header: http.Header{
+					"Authorization":        []string{"AWS4-HMAC-SHA256 <cred_string>"},
+					"X-Amz-Content-Sha256": []string{streamingContentSHA256},
+				},
+				Method: "PUT",
+			},
+			authT: authTypeStreamingSigned,
+		},
+		// Test case - 2
+		// Check for JWT header.
+		{
+			req: &http.Request{
+				URL: &url.URL{
+					Host:   "localhost:9000",
+					Scheme: "http",
+					Path:   "/",
+				},
+				Header: http.Header{
+					"Authorization": []string{"Bearer 12313123"},
+				},
+			},
+			authT: authTypeJWT,
+		},
+		// Test case - 3
+		// Empty authorization header.
+		{
+			req: &http.Request{
+				URL: &url.URL{
+					Host:   "localhost:9000",
+					Scheme: "http",
+					Path:   "/",
+				},
+				Header: http.Header{
+					"Authorization": []string{""},
+				},
+			},
+			authT: authTypeUnknown,
+		},
+		// Test case - 4
+		// Check for presigned.
+		{
+			req: &http.Request{
+				URL: &url.URL{
+					Host:     "localhost:9000",
+					Scheme:   "http",
+					Path:     "/",
+					RawQuery: "X-Amz-Credential=EXAMPLEINVALIDEXAMPL%2Fs3%2F20160314%2Fus-east-1",
+				},
+			},
+			authT: authTypePresigned,
+		},
+		// Test case - 5
+		// Check for post policy.
+		{
+			req: &http.Request{
+				URL: &url.URL{
+					Host:   "localhost:9000",
+					Scheme: "http",
+					Path:   "/",
+				},
+				Header: http.Header{
+					"Content-Type": []string{"multipart/form-data"},
+				},
+				Method: "POST",
+			},
+			authT: authTypePostPolicy,
+		},
+	}
+
+	// .. Tests all request auth type.
+	for i, testc := range testCases {
+		authT := getRequestAuthType(testc.req)
+		if authT != testc.authT {
+			t.Errorf("Test %d: Expected %d, got %d", i+1, testc.authT, authT)
+		}
+	}
+}
+
+// Test all s3 supported auth types.
+func TestS3SupportedAuthType(t *testing.T) {
+	type testCase struct {
+		authT authType
+		pass  bool
+	}
+	// List of all valid and invalid test cases.
+	testCases := []testCase{
+		// Test 1 - supported s3 type anonymous.
+		{
+			authT: authTypeAnonymous,
+			pass:  true,
+		},
+		// Test 2 - supported s3 type presigned.
+		{
+			authT: authTypePresigned,
+			pass:  true,
+		},
+		// Test 3 - supported s3 type signed.
+		{
+			authT: authTypeSigned,
+			pass:  true,
+		},
+		// Test 4 - supported s3 type with post policy.
+		{
+			authT: authTypePostPolicy,
+			pass:  true,
+		},
+		// Test 5 - supported s3 type with streaming signed.
+		{
+			authT: authTypeStreamingSigned,
+			pass:  true,
+		},
+		// Test 6 - supported s3 type with signature v2.
+		{
+			authT: authTypeSignedV2,
+			pass:  true,
+		},
+		// Test 7 - supported s3 type with presign v2.
+		{
+			authT: authTypePresignedV2,
+			pass:  true,
+		},
+		// Test 8 - JWT is not supported s3 type.
+		{
+			authT: authTypeJWT,
+			pass:  false,
+		},
+		// Test 9 - unknown auth header is not supported s3 type.
+		{
+			authT: authTypeUnknown,
+			pass:  false,
+		},
+		// Test 10 - some new auth type is not supported s3 type.
+		{
+			authT: authType(9),
+			pass:  false,
+		},
+	}
+	// Validate all the test cases.
+	for i, tt := range testCases {
+		ok := isSupportedS3AuthType(tt.authT)
+		if ok != tt.pass {
+			t.Errorf("Test %d:, Expected %t, got %t", i+1, tt.pass, ok)
+		}
+	}
+}
+
+// TestIsRequestUnsignedPayload - Test validates the Unsigned payload detection logic.
+func TestIsRequestUnsignedPayload(t *testing.T) {
+	testCases := []struct {
+		inputAmzContentHeader string
+		expectedResult        bool
+	}{
+		// Test case - 1.
+		// Test case with "X-Amz-Content-Sha256" header set to empty value.
+		{"", false},
+		// Test case - 2.
+		// Test case with "X-Amz-Content-Sha256" header set to  "UNSIGNED-PAYLOAD"
+		// The payload is flagged as unsigned When "X-Amz-Content-Sha256" header is set to  "UNSIGNED-PAYLOAD".
+		{unsignedPayload, true},
+		// Test case - 3.
+		// set to a random value.
+		{"abcd", false},
+	}
+
+	// creating an input HTTP request.
+	// Only the headers are relevant for this particular test.
+	inputReq, err := http.NewRequest("GET", "http://example.com", nil)
+	if err != nil {
+		t.Fatalf("Error initializing input HTTP request: %v", err)
+	}
+
+	for i, testCase := range testCases {
+		inputReq.Header.Set("X-Amz-Content-Sha256", testCase.inputAmzContentHeader)
+		actualResult := isRequestUnsignedPayload(inputReq)
+		if testCase.expectedResult != actualResult {
+			t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult)
+		}
+	}
+}
+
+func TestIsRequestPresignedSignatureV2(t *testing.T) {
+	testCases := []struct {
+		inputQueryKey   string
+		inputQueryValue string
+		expectedResult  bool
+	}{
+		// Test case - 1.
+		// Test case with query key "AWSAccessKeyId" set.
+		{"", "", false},
+		// Test case - 2.
+		{"AWSAccessKeyId", "", true},
+		// Test case - 3.
+		{"X-Amz-Content-Sha256", "", false},
+	}
+
+	for i, testCase := range testCases {
+		// creating an input HTTP request.
+		// Only the query parameters are relevant for this particular test.
+		inputReq, err := http.NewRequest("GET", "http://example.com", nil)
+		if err != nil {
+			t.Fatalf("Error initializing input HTTP request: %v", err)
+		}
+		q := inputReq.URL.Query()
+		q.Add(testCase.inputQueryKey, testCase.inputQueryValue)
+		inputReq.URL.RawQuery = q.Encode()
+
+		actualResult := isRequestPresignedSignatureV2(inputReq)
+		if testCase.expectedResult != actualResult {
+			t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult)
+		}
+	}
+}
+
+// TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature verision v4 detection.
+func TestIsRequestPresignedSignatureV4(t *testing.T) {
+	testCases := []struct {
+		inputQueryKey   string
+		inputQueryValue string
+		expectedResult  bool
+	}{
+		// Test case - 1.
+		// Test case with query key ""X-Amz-Credential" set.
+		{"", "", false},
+		// Test case - 2.
+		{"X-Amz-Credential", "", true},
+		// Test case - 3.
+		{"X-Amz-Content-Sha256", "", false},
+	}
+
+	for i, testCase := range testCases {
+		// creating an input HTTP request.
+		// Only the query parameters are relevant for this particular test.
+		inputReq, err := http.NewRequest("GET", "http://example.com", nil)
+		if err != nil {
+			t.Fatalf("Error initializing input HTTP request: %v", err)
+		}
+		q := inputReq.URL.Query()
+		q.Add(testCase.inputQueryKey, testCase.inputQueryValue)
+		inputReq.URL.RawQuery = q.Encode()
+
+		actualResult := isRequestPresignedSignatureV4(inputReq)
+		if testCase.expectedResult != actualResult {
+			t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult)
+		}
+	}
+}
+
+// Provides a fully populated http request instance, fails otherwise.
+func mustNewRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+	req, err := newTestRequest(method, urlStr, contentLength, body)
+	if err != nil {
+		t.Fatalf("Unable to initialize new http request %s", err)
+	}
+	return req
+}
+
+// This is similar to mustNewRequest but additionally the request
+// is signed with AWS Signature V4, fails if not able to do so.
+func mustNewSignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+	req := mustNewRequest(method, urlStr, contentLength, body, t)
+	cred := serverConfig.GetCredential()
+	if err := signRequestV4(req, cred.AccessKeyID, cred.SecretAccessKey); err != nil {
+		t.Fatalf("Unable to inititalized new signed http request %s", err)
+	}
+	return req
+}
+
+// Tests is requested authenticated function, tests replies for s3 errors.
+func TestIsReqAuthenticated(t *testing.T) {
+	path, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(path)
+
+	serverConfig.SetCredential(credential{"myuser", "mypassword"})
+
+	// List of test cases for validating http request authentication.
+	testCases := []struct {
+		req     *http.Request
+		s3Error APIErrorCode
+	}{
+		// When request is nil, internal error is returned.
+		{nil, ErrInternalError},
+		// When request is unsigned, access denied is returned.
+		{mustNewRequest("GET", "http://localhost:9000", 0, nil, t), ErrAccessDenied},
+		// When request is properly signed, but has bad Content-MD5 header.
+		{mustNewSignedRequest("PUT", "http://localhost:9000", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
+		// When request is properly signed, error is none.
+		{mustNewSignedRequest("GET", "http://localhost:9000", 0, nil, t), ErrNone},
+	}
+
+	// Validates all testcases.
+	for _, testCase := range testCases {
+		if testCase.s3Error == ErrBadDigest {
+			testCase.req.Header.Set("Content-Md5", "garbage")
+		}
+		if s3Error := isReqAuthenticated(testCase.req, serverConfig.GetRegion()); s3Error != testCase.s3Error {
+			t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
+		}
+	}
+}

cmd/auth-rpc-client.go

@@ -0,0 +1,206 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net/rpc"
+	"sync"
+	"time"
+
+	jwtgo "github.com/dgrijalva/jwt-go"
+)
+
+// GenericReply represents any generic RPC reply.
+type GenericReply struct{}
+
+// GenericArgs represents any generic RPC arguments.
+type GenericArgs struct {
+	Token string // Used to authenticate every RPC call.
+	// Used to verify if the RPC call was issued between
+	// the same Login() and disconnect event pair.
+	Timestamp time.Time
+
+	// Indicates if args should be sent to remote peers as well.
+	Remote bool
+}
+
+// SetToken - sets the token to the supplied value.
+func (ga *GenericArgs) SetToken(token string) {
+	ga.Token = token
+}
+
+// SetTimestamp - sets the timestamp to the supplied value.
+func (ga *GenericArgs) SetTimestamp(tstamp time.Time) {
+	ga.Timestamp = tstamp
+}
+
+// RPCLoginArgs - login username and password for RPC.
+type RPCLoginArgs struct {
+	Username string
+	Password string
+}
+
+// RPCLoginReply - login reply provides generated token to be used
+// with subsequent requests.
+type RPCLoginReply struct {
+	Token         string
+	Timestamp     time.Time
+	ServerVersion string
+}
+
+// Validates if incoming token is valid.
+func isRPCTokenValid(tokenStr string) bool {
+	jwt, err := newJWT(defaultInterNodeJWTExpiry, serverConfig.GetCredential())
+	if err != nil {
+		errorIf(err, "Unable to initialize JWT")
+		return false
+	}
+	token, err := jwtgo.Parse(tokenStr, func(token *jwtgo.Token) (interface{}, error) {
+		if _, ok := token.Method.(*jwtgo.SigningMethodHMAC); !ok {
+			return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
+		}
+		return []byte(jwt.SecretAccessKey), nil
+	})
+	if err != nil {
+		errorIf(err, "Unable to parse JWT token string")
+		return false
+	}
+	// Return if token is valid.
+	return token.Valid
+}
+
+// Auth config represents authentication credentials and Login method name to be used
+// for fetching JWT tokens from the RPC server.
+type authConfig struct {
+	accessKey   string // Username for the server.
+	secretKey   string // Password for the server.
+	secureConn  bool   // Ask for a secured connection
+	address     string // Network address path of RPC server.
+	path        string // Network path for HTTP dial.
+	loginMethod string // RPC service name for authenticating using JWT
+}
+
+// AuthRPCClient is a wrapper type for RPCClient which provides JWT based authentication across reconnects.
+type AuthRPCClient struct {
+	mu            sync.Mutex
+	config        *authConfig
+	rpc           *RPCClient // reconnect'able rpc client built on top of net/rpc Client
+	isLoggedIn    bool       // Indicates if the auth client has been logged in and token is valid.
+	serverToken   string     // Disk rpc JWT based token.
+	serverVersion string     // Server version exchanged by the RPC.
+}
+
+// newAuthClient - returns a jwt based authenticated (go) rpc client, which does automatic reconnect.
+func newAuthClient(cfg *authConfig) *AuthRPCClient {
+	return &AuthRPCClient{
+		// Save the config.
+		config: cfg,
+		// Initialize a new reconnectable rpc client.
+		rpc: newClient(cfg.address, cfg.path, cfg.secureConn),
+		// Allocated auth client not logged in yet.
+		isLoggedIn: false,
+	}
+}
+
+// Close - closes underlying rpc connection.
+func (authClient *AuthRPCClient) Close() error {
+	authClient.mu.Lock()
+	// reset token on closing a connection
+	authClient.isLoggedIn = false
+	authClient.mu.Unlock()
+	return authClient.rpc.Close()
+}
+
+// Login - a jwt based authentication is performed with rpc server.
+func (authClient *AuthRPCClient) Login() (err error) {
+	authClient.mu.Lock()
+	// As soon as the function returns unlock,
+	defer authClient.mu.Unlock()
+
+	// Return if already logged in.
+	if authClient.isLoggedIn {
+		return nil
+	}
+
+	reply := RPCLoginReply{}
+	if err = authClient.rpc.Call(authClient.config.loginMethod, RPCLoginArgs{
+		Username: authClient.config.accessKey,
+		Password: authClient.config.secretKey,
+	}, &reply); err != nil {
+		return err
+	}
+
+	// Validate if version do indeed match.
+	if reply.ServerVersion != Version {
+		return errServerVersionMismatch
+	}
+
+	// Validate if server timestamp is skewed.
+	curTime := time.Now().UTC()
+	if curTime.Sub(reply.Timestamp) > globalMaxSkewTime {
+		return errServerTimeMismatch
+	}
+
+	// Set token, time stamp as received from a successful login call.
+	authClient.serverToken = reply.Token
+	authClient.serverVersion = reply.ServerVersion
+	authClient.isLoggedIn = true
+	return nil
+}
+
+// Call - If rpc connection isn't established yet since previous disconnect,
+// connection is established, a jwt authenticated login is performed and then
+// the call is performed.
+func (authClient *AuthRPCClient) Call(serviceMethod string, args interface {
+	SetToken(token string)
+	SetTimestamp(tstamp time.Time)
+}, reply interface{}) (err error) {
+	// On successful login, attempt the call.
+	if err = authClient.Login(); err == nil {
+		// Set token and timestamp before the rpc call.
+		args.SetToken(authClient.serverToken)
+		args.SetTimestamp(time.Now().UTC())
+
+		// Call the underlying rpc.
+		err = authClient.rpc.Call(serviceMethod, args, reply)
+
+		// Invalidate token, and mark it for re-login on subsequent reconnect.
+		if err == rpc.ErrShutdown {
+			authClient.mu.Lock()
+			authClient.isLoggedIn = false
+			authClient.mu.Unlock()
+		}
+	}
+	return err
+}
+
+// Node returns the node (network address) of the connection
+func (authClient *AuthRPCClient) Node() (node string) {
+	if authClient.rpc != nil {
+		node = authClient.rpc.node
+	}
+	return node
+}
+
+// RPCPath returns the RPC path of the connection
+func (authClient *AuthRPCClient) RPCPath() (rpcPath string) {
+	if authClient.rpc != nil {
+		rpcPath = authClient.rpc.rpcPath
+	}
+	return rpcPath
+}

cmd/auth-rpc-client_test.go

@@ -0,0 +1,51 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "testing"
+
+// Tests authorized RPC client.
+func TestAuthRPCClient(t *testing.T) {
+	authCfg := &authConfig{
+		accessKey:   "123",
+		secretKey:   "123",
+		secureConn:  false,
+		address:     "localhost:9000",
+		path:        "/rpc/disk",
+		loginMethod: "MyPackage.LoginHandler",
+	}
+	authRPC := newAuthClient(authCfg)
+	if authRPC.Node() != authCfg.address {
+		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.Node(), authCfg.address)
+	}
+	if authRPC.RPCPath() != authCfg.path {
+		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.RPCPath(), authCfg.path)
+	}
+	authCfg = &authConfig{
+		accessKey:   "123",
+		secretKey:   "123",
+		secureConn:  false,
+		loginMethod: "MyPackage.LoginHandler",
+	}
+	authRPC = newAuthClient(authCfg)
+	if authRPC.Node() != authCfg.address {
+		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.Node(), authCfg.address)
+	}
+	if authRPC.RPCPath() != authCfg.path {
+		t.Fatalf("Unexpected node value %s, but expected %s", authRPC.RPCPath(), authCfg.path)
+	}
+}

cmd/benchmark-utils_test.go

@@ -0,0 +1,428 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"io/ioutil"
+	"math"
+	"math/rand"
+	"strconv"
+	"testing"
+	"time"
+
+	humanize "github.com/dustin/go-humanize"
+)
+
+// Prepare benchmark backend
+func prepareBenchmarkBackend(instanceType string) (ObjectLayer, []string, error) {
+	var nDisks int
+	switch instanceType {
+	// Total number of disks for FS backend is set to 1.
+	case FSTestStr:
+		nDisks = 1
+		// Total number of disks for FS backend is set to 16.
+	case XLTestStr:
+		nDisks = 16
+	default:
+		nDisks = 1
+	}
+	// get `nDisks` random disks.
+	disks, err := getRandomDisks(nDisks)
+	if err != nil {
+		return nil, nil, err
+	}
+	endpoints, err := parseStorageEndpoints(disks)
+	if err != nil {
+		return nil, nil, err
+	}
+	// initialize object layer.
+	obj, _, err := initObjectLayer(endpoints)
+	if err != nil {
+		return nil, nil, err
+	}
+	return obj, disks, nil
+}
+
+// Benchmark utility functions for ObjectLayer.PutObject().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
+func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for PutObject.
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = getMD5Hash(textData)
+	sha256sum := ""
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// insert the object.
+		objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if objInfo.MD5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// Benchmark utility functions for ObjectLayer.PutObjectPart().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObjectPart benchmark.
+func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	objSize := 128 * humanize.MiByte
+
+	// PutObjectPart returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum, uploadID string
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for NewMultipartUpload.
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = getMD5Hash(textData)
+	sha256sum := ""
+	uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	var textPartData []byte
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObjectPart starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// insert the object.
+		totalPartsNR := int(math.Ceil(float64(objSize) / float64(partSize)))
+		for j := 0; j < totalPartsNR; j++ {
+			if j < totalPartsNR-1 {
+				textPartData = textData[j*partSize : (j+1)*partSize-1]
+			} else {
+				textPartData = textData[j*partSize:]
+			}
+			metadata := make(map[string]string)
+			metadata["md5Sum"] = getMD5Hash([]byte(textPartData))
+			md5Sum, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"], sha256sum)
+			if err != nil {
+				b.Fatal(err)
+			}
+			if md5Sum != metadata["md5Sum"] {
+				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+			}
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
+func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		b.Fatalf("Unable to initialize config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// create a temp XL/FS backend.
+	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// uses *testing.B and the object Layer to run the benchmark.
+	runPutObjectPartBenchmark(b, objLayer, objSize)
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
+func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		b.Fatalf("Unable to initialize config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// create a temp XL/FS backend.
+	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// uses *testing.B and the object Layer to run the benchmark.
+	runPutObjectBenchmark(b, objLayer, objSize)
+}
+
+// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
+func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		b.Fatalf("Unable to initialize config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// create a temp XL/FS backend.
+	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// uses *testing.B and the object Layer to run the benchmark.
+	runPutObjectBenchmarkParallel(b, objLayer, objSize)
+}
+
+// Benchmark utility functions for ObjectLayer.GetObject().
+// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
+func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		b.Fatalf("Unable to initialize config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	sha256sum := ""
+	for i := 0; i < 10; i++ {
+		// get text data generated for number of bytes equal to object size.
+		textData := generateBytesData(objSize)
+		// generate md5sum for the generated data.
+		// md5sum of the data to written is required as input for PutObject.
+		// PutObject is the functions which writes the data onto the FS/XL backend.
+		metadata := make(map[string]string)
+		metadata["md5Sum"] = getMD5Hash(textData)
+		// insert the object.
+		var objInfo ObjectInfo
+		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if objInfo.MD5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		}
+	}
+
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for GetObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		var buffer = new(bytes.Buffer)
+		err = obj.GetObject(bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer)
+		if err != nil {
+			b.Error(err)
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+
+}
+
+// randomly picks a character and returns its equivalent byte array.
+func getRandomByte() []byte {
+	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	// seeding the random number generator.
+	rand.Seed(time.Now().UTC().UnixNano())
+	var b byte
+	// pick a character randomly.
+	b = letterBytes[rand.Intn(len(letterBytes))]
+	return []byte{b}
+}
+
+// picks a random byte and repeats it to size bytes.
+func generateBytesData(size int) []byte {
+	// repeat the random character chosen size
+	return bytes.Repeat(getRandomByte(), size)
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
+func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		b.Fatalf("Unable to initialize config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// create a temp XL/FS backend.
+	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created.
+	defer removeRoots(disks)
+	//  uses *testing.B and the object Layer to run the benchmark.
+	runGetObjectBenchmark(b, objLayer, objSize)
+}
+
+// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
+func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		b.Fatalf("Unable to initialize config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// create a temp XL/FS backend.
+	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created.
+	defer removeRoots(disks)
+	//  uses *testing.B and the object Layer to run the benchmark.
+	runGetObjectBenchmarkParallel(b, objLayer, objSize)
+}
+
+// Parallel benchmark utility functions for ObjectLayer.PutObject().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
+func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		b.Fatalf("Unable to initialize config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for PutObject.
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = getMD5Hash([]byte(textData))
+	sha256sum := ""
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		i := 0
+		for pb.Next() {
+			// insert the object.
+			objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+			if err != nil {
+				b.Fatal(err)
+			}
+			if objInfo.MD5Sum != metadata["md5Sum"] {
+				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.MD5Sum, metadata["md5Sum"])
+			}
+			i++
+		}
+	})
+
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// Parallel benchmark utility functions for ObjectLayer.GetObject().
+// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
+func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		b.Fatalf("Unable to initialize config. %s", err)
+	}
+	defer removeAll(rootPath)
+
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	for i := 0; i < 10; i++ {
+		// get text data generated for number of bytes equal to object size.
+		textData := generateBytesData(objSize)
+		// generate md5sum for the generated data.
+		// md5sum of the data to written is required as input for PutObject.
+		// PutObject is the functions which writes the data onto the FS/XL backend.
+		metadata := make(map[string]string)
+		metadata["md5Sum"] = getMD5Hash([]byte(textData))
+		sha256sum := ""
+		// insert the object.
+		var objInfo ObjectInfo
+		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if objInfo.MD5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		}
+	}
+
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for GetObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		i := 0
+		for pb.Next() {
+			err = obj.GetObject(bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard)
+			if err != nil {
+				b.Error(err)
+			}
+			i++
+			if i == 10 {
+				i = 0
+			}
+		}
+	})
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+
+}

cmd/browser-peer-rpc.go

@@ -0,0 +1,150 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"path"
+	"sync"
+	"time"
+)
+
+// Login handler implements JWT login token generator, which upon login request
+// along with username and password is generated.
+func (br *browserPeerAPIHandlers) LoginHandler(args *RPCLoginArgs, reply *RPCLoginReply) error {
+	jwt, err := newJWT(defaultInterNodeJWTExpiry, serverConfig.GetCredential())
+	if err != nil {
+		return err
+	}
+	if err = jwt.Authenticate(args.Username, args.Password); err != nil {
+		return err
+	}
+	token, err := jwt.GenerateToken(args.Username)
+	if err != nil {
+		return err
+	}
+	reply.Token = token
+	reply.ServerVersion = Version
+	reply.Timestamp = time.Now().UTC()
+	return nil
+}
+
+// SetAuthPeerArgs - Arguments collection for SetAuth RPC call
+type SetAuthPeerArgs struct {
+	// For Auth
+	GenericArgs
+
+	// New credentials that receiving peer should update to.
+	Creds credential
+}
+
+// SetAuthPeer - Update to new credentials sent from a peer Minio
+// server. Since credentials are already validated on the sending
+// peer, here we just persist to file and update in-memory config. All
+// subsequently running isRPCTokenValid() calls will fail, and clients
+// will be forced to re-establish connections. Connections will be
+// re-established only when the sending client has also updated its
+// credentials.
+func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *GenericReply) error {
+	// Check auth
+	if !isRPCTokenValid(args.Token) {
+		return errInvalidToken
+	}
+
+	// Update credentials in memory
+	serverConfig.SetCredential(args.Creds)
+
+	// Save credentials to config file
+	if err := serverConfig.Save(); err != nil {
+		errorIf(err, "Error updating config file with new credentials sent from browser RPC.")
+		return err
+	}
+
+	return nil
+}
+
+// Sends SetAuthPeer RPCs to all peers in the Minio cluster
+func updateCredsOnPeers(creds credential) map[string]error {
+	// Get list of peer addresses (from globalS3Peers)
+	peers := []string{}
+	for _, p := range globalS3Peers {
+		peers = append(peers, p.addr)
+	}
+
+	// Array of errors for each peer
+	errs := make([]error, len(peers))
+	var wg sync.WaitGroup
+
+	// Launch go routines to send request to each peer in parallel.
+	for ix := range peers {
+		wg.Add(1)
+		go func(ix int) {
+			defer wg.Done()
+
+			// Exclude self to avoid race with
+			// invalidating the RPC token.
+			if peers[ix] == globalMinioAddr {
+				errs[ix] = nil
+				return
+			}
+
+			// Initialize client
+			client := newAuthClient(&authConfig{
+				accessKey:   serverConfig.GetCredential().AccessKeyID,
+				secretKey:   serverConfig.GetCredential().SecretAccessKey,
+				address:     peers[ix],
+				secureConn:  isSSL(),
+				path:        path.Join(reservedBucket, browserPeerPath),
+				loginMethod: "Browser.LoginHandler",
+			})
+
+			// Construct RPC call arguments.
+			args := SetAuthPeerArgs{Creds: creds}
+
+			// Make RPC call - we only care about error
+			// response and not the reply.
+			err := client.Call("Browser.SetAuthPeer", &args, &GenericReply{})
+
+			// We try a bit hard (3 attempts with 1 second delay)
+			// to set creds on peers in case of failure.
+			if err != nil {
+				for i := 0; i < 2; i++ {
+					time.Sleep(1 * time.Second) // 1 second delay.
+					err = client.Call("Browser.SetAuthPeer", &args, &GenericReply{})
+					if err == nil {
+						break
+					}
+				}
+			}
+
+			// Send result down the channel
+			errs[ix] = err
+		}(ix)
+	}
+
+	// Wait for requests to complete.
+	wg.Wait()
+
+	// Put errors into map.
+	errsMap := make(map[string]error)
+	for i, err := range errs {
+		if err != nil {
+			errsMap[peers[i]] = err
+		}
+	}
+
+	return errsMap
+}

cmd/browser-peer-rpc_test.go

@@ -0,0 +1,123 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"path"
+	"testing"
+)
+
+// API suite container common to both FS and XL.
+type TestRPCBrowserPeerSuite struct {
+	serverType   string
+	testServer   TestServer
+	testAuthConf *authConfig
+}
+
+// Setting up the test suite and starting the Test server.
+func (s *TestRPCBrowserPeerSuite) SetUpSuite(c *testing.T) {
+	s.testServer = StartTestBrowserPeerRPCServer(c, s.serverType)
+	s.testAuthConf = &authConfig{
+		address:     s.testServer.Server.Listener.Addr().String(),
+		accessKey:   s.testServer.AccessKey,
+		secretKey:   s.testServer.SecretKey,
+		path:        path.Join(reservedBucket, browserPeerPath),
+		loginMethod: "BrowserPeer.LoginHandler",
+	}
+}
+
+// No longer used with gocheck, but used in explicit teardown code in
+// each test function. // Called implicitly by "gopkg.in/check.v1"
+// after all tests are run.
+func (s *TestRPCBrowserPeerSuite) TearDownSuite(c *testing.T) {
+	s.testServer.Stop()
+}
+
+func TestBrowserPeerRPC(t *testing.T) {
+	// setup code
+	s := &TestRPCBrowserPeerSuite{serverType: "XL"}
+	s.SetUpSuite(t)
+
+	// run test
+	s.testBrowserPeerRPC(t)
+
+	// teardown code
+	s.TearDownSuite(t)
+}
+
+// Tests for browser peer rpc.
+func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
+	// Construct RPC call arguments.
+	creds := credential{
+		AccessKeyID:     "abcd1",
+		SecretAccessKey: "abcd1234",
+	}
+
+	// Validate for invalid token.
+	args := SetAuthPeerArgs{Creds: creds}
+	args.Token = "garbage"
+	rclient := newClient(s.testAuthConf.address, s.testAuthConf.path, false)
+	defer rclient.Close()
+	err := rclient.Call("BrowserPeer.SetAuthPeer", &args, &GenericReply{})
+	if err != nil {
+		if err.Error() != errInvalidToken.Error() {
+			t.Fatal(err)
+		}
+	}
+
+	// Validate for successful Peer update.
+	args = SetAuthPeerArgs{Creds: creds}
+	client := newAuthClient(s.testAuthConf)
+	defer client.Close()
+	err = client.Call("BrowserPeer.SetAuthPeer", &args, &GenericReply{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Validate for failure in login handler with previous credentials.
+	rclient = newClient(s.testAuthConf.address, s.testAuthConf.path, false)
+	defer rclient.Close()
+	rargs := &RPCLoginArgs{
+		Username: s.testAuthConf.accessKey,
+		Password: s.testAuthConf.secretKey,
+	}
+	rreply := &RPCLoginReply{}
+	err = rclient.Call("BrowserPeer.LoginHandler", rargs, rreply)
+	if err != nil {
+		if err.Error() != errInvalidAccessKeyID.Error() {
+			t.Fatal(err)
+		}
+	}
+
+	// Validate for success in loing handled with valid credetnails.
+	rargs = &RPCLoginArgs{
+		Username: creds.AccessKeyID,
+		Password: creds.SecretAccessKey,
+	}
+	rreply = &RPCLoginReply{}
+	err = rclient.Call("BrowserPeer.LoginHandler", rargs, rreply)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Validate all the replied fields after successful login.
+	if rreply.Token == "" {
+		t.Fatalf("Generated token cannot be empty %s", errInvalidToken)
+	}
+	if rreply.Timestamp.IsZero() {
+		t.Fatal("Time stamp returned cannot be zero")
+	}
+}

cmd/browser-rpc-router.go

@@ -0,0 +1,49 @@
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/rpc"
+
+	router "github.com/gorilla/mux"
+)
+
+// Set up an RPC endpoint that receives browser related calls. The
+// original motivation is for propagating credentials change
+// throughout Minio cluster, initiated from a Minio browser session.
+
+const (
+	browserPeerPath = "/browser/setauth"
+)
+
+// The Type exporting methods exposed for RPC calls.
+type browserPeerAPIHandlers struct{}
+
+// Register RPC router
+func registerBrowserPeerRPCRouter(mux *router.Router) error {
+	bpHandlers := &browserPeerAPIHandlers{}
+
+	bpRPCServer := rpc.NewServer()
+	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
+	if err != nil {
+		return traceError(err)
+	}
+
+	bpRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter()
+	bpRouter.Path(browserPeerPath).Handler(bpRPCServer)
+	return nil
+}

cmd/bucket-handlers-listobjects.go

@@ -0,0 +1,155 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/gorilla/mux"
+)
+
+// Validate all the ListObjects query arguments, returns an APIErrorCode
+// if one of the args do not meet the required conditions.
+// Special conditions required by Minio server are as below
+// - delimiter if set should be equal to '/', otherwise the request is rejected.
+// - marker if set should have a common prefix with 'prefix' param, otherwise
+//   the request is rejected.
+func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIErrorCode {
+	// Max keys cannot be negative.
+	if maxKeys < 0 {
+		return ErrInvalidMaxKeys
+	}
+
+	/// Minio special conditions for ListObjects.
+
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		return ErrNotImplemented
+	}
+	// Marker is set validate pre-condition.
+	if marker != "" {
+		// Marker not common with prefix is not implemented.
+		if !strings.HasPrefix(marker, prefix) {
+			return ErrNotImplemented
+		}
+	}
+	// Success.
+	return ErrNone
+}
+
+// ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
+// --------------------------
+// This implementation of the GET operation returns some or all (up to 1000)
+// of the objects in a bucket. You can use the request parameters as selection
+// criteria to return a subset of the objects in a bucket.
+//
+// NOTE: It is recommended that this API to be used for application development.
+// Minio continues to support ListObjectsV1 for supporting legacy tools.
+func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Extract all the listObjectsV2 query params to their native values.
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+
+	// In ListObjectsV2 'continuation-token' is the marker.
+	marker := token
+	// Check if 'continuation-token' is empty.
+	if token == "" {
+		// Then we need to use 'start-after' as marker instead.
+		marker = startAfter
+	}
+	// Validate the query params before beginning to serve the request.
+	// fetch-owner is not validated since it is a boolean
+	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+	// Inititate a list objects operation based on the input params.
+	// On success would return back ListObjectsInfo object to be
+	// marshalled into S3 compatible XML header.
+	listObjectsInfo, err := objectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
+	if err != nil {
+		errorIf(err, "Unable to list objects.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, fetchOwner, maxKeys, listObjectsInfo)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodeResponse(response))
+}
+
+// ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
+// --------------------------
+// This implementation of the GET operation returns some or all (up to 1000)
+// of the objects in a bucket. You can use the request parameters as selection
+// criteria to return a subset of the objects in a bucket.
+//
+func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Extract all the litsObjectsV1 query params to their native values.
+	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
+
+	// Validate all the query params before beginning to serve the request.
+	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Inititate a list objects operation based on the input params.
+	// On success would return back ListObjectsInfo object to be
+	// marshalled into S3 compatible XML header.
+	listObjectsInfo, err := objectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
+	if err != nil {
+		errorIf(err, "Unable to list objects.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodeResponse(response))
+}

cmd/bucket-handlers.go

@@ -0,0 +1,540 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/base64"
+	"encoding/xml"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+
+	mux "github.com/gorilla/mux"
+	"github.com/minio/minio-go/pkg/set"
+)
+
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+// Enforces bucket policies for a bucket for a given tatusaction.
+func enforceBucketPolicy(bucket string, action string, reqURL *url.URL) (s3Error APIErrorCode) {
+	// Verify if bucket actually exists
+	if err := checkBucketExist(bucket, newObjectLayerFn()); err != nil {
+		err = errorCause(err)
+		switch err.(type) {
+		case BucketNameInvalid:
+			// Return error for invalid bucket name.
+			return ErrInvalidBucketName
+		case BucketNotFound:
+			// For no bucket found we return NoSuchBucket instead.
+			return ErrNoSuchBucket
+		}
+		errorIf(err, "Unable to read bucket policy.")
+		// Return internal error for any other errors so that we can investigate.
+		return ErrInternalError
+	}
+
+	// Fetch bucket policy, if policy is not set return access denied.
+	policy := globalBucketPolicies.GetBucketPolicy(bucket)
+	if policy == nil {
+		return ErrAccessDenied
+	}
+
+	// Construct resource in 'arn:aws:s3:::examplebucket/object' format.
+	resource := AWSResourcePrefix + strings.TrimSuffix(strings.TrimPrefix(reqURL.Path, "/"), "/")
+
+	// Get conditions for policy verification.
+	conditionKeyMap := make(map[string]set.StringSet)
+	for queryParam := range reqURL.Query() {
+		conditionKeyMap[queryParam] = set.CreateStringSet(reqURL.Query().Get(queryParam))
+	}
+
+	// Validate action, resource and conditions with current policy statements.
+	if !bucketPolicyEvalStatements(action, resource, conditionKeyMap, policy.Statements) {
+		return ErrAccessDenied
+	}
+	return ErrNone
+}
+
+// GetBucketLocationHandler - GET Bucket location.
+// -------------------------
+// This operation returns bucket location.
+func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, bucket, "s3:GetBucketLocation", "us-east-1"); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	if _, err := objectAPI.GetBucketInfo(bucket); err != nil {
+		errorIf(err, "Unable to fetch bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Generate response.
+	encodedSuccessResponse := encodeResponse(LocationResponse{})
+	// Get current region.
+	region := serverConfig.GetRegion()
+	if region != "us-east-1" {
+		encodedSuccessResponse = encodeResponse(LocationResponse{
+			Location: region,
+		})
+	}
+	setCommonHeaders(w) // Write headers.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// ListMultipartUploadsHandler - GET Bucket (List Multipart uploads)
+// -------------------------
+// This operation lists in-progress multipart uploads. An in-progress
+// multipart upload is a multipart upload that has been initiated,
+// using the Initiate Multipart Upload request, but has not yet been
+// completed or aborted. This operation returns at most 1,000 multipart
+// uploads in the response.
+//
+func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
+	if maxUploads < 0 {
+		writeErrorResponse(w, r, ErrInvalidMaxUploads, r.URL.Path)
+		return
+	}
+	if keyMarker != "" {
+		// Marker not common with prefix is not implemented.
+		if !strings.HasPrefix(keyMarker, prefix) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
+	}
+
+	listMultipartsInfo, err := objectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
+	if err != nil {
+		errorIf(err, "Unable to list multipart uploads.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// generate response
+	response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo)
+	encodedSuccessResponse := encodeResponse(response)
+	// write headers.
+	setCommonHeaders(w)
+	// write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// ListBucketsHandler - GET Service.
+// -----------
+// This implementation of the GET operation returns a list of all buckets
+// owned by the authenticated sender of the request.
+func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	// ListBuckets does not have any bucket action.
+	s3Error := checkRequestAuthType(r, "", "", "us-east-1")
+	if s3Error == ErrInvalidRegion {
+		// Clients like boto3 send listBuckets() call signed with region that is configured.
+		s3Error = checkRequestAuthType(r, "", "", serverConfig.GetRegion())
+	}
+	if s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+	// Invoke the list buckets.
+	bucketsInfo, err := objectAPI.ListBuckets()
+	if err != nil {
+		errorIf(err, "Unable to list buckets.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Generate response.
+	response := generateListBucketsResponse(bucketsInfo)
+	encodedSuccessResponse := encodeResponse(response)
+	// Write headers.
+	setCommonHeaders(w)
+	// Write response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// DeleteMultipleObjectsHandler - deletes multiple objects.
+func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Content-Length is required and should be non-zero
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if r.ContentLength <= 0 {
+		writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+		return
+	}
+
+	// Content-Md5 is requied should be set
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if _, ok := r.Header["Content-Md5"]; !ok {
+		writeErrorResponse(w, r, ErrMissingContentMD5, r.URL.Path)
+		return
+	}
+
+	// Allocate incoming content length bytes.
+	deleteXMLBytes := make([]byte, r.ContentLength)
+
+	// Read incoming body XML bytes.
+	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
+		errorIf(err, "Unable to read HTTP body.")
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
+
+	// Unmarshal list of keys to be deleted.
+	deleteObjects := &DeleteObjectsRequest{}
+	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
+		errorIf(err, "Unable to unmarshal delete objects request XML.")
+		writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
+		return
+	}
+
+	var wg = &sync.WaitGroup{} // Allocate a new wait group.
+	var dErrs = make([]error, len(deleteObjects.Objects))
+
+	// Delete all requested objects in parallel.
+	for index, object := range deleteObjects.Objects {
+		wg.Add(1)
+		go func(i int, obj ObjectIdentifier) {
+			defer wg.Done()
+			dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
+			if dErr != nil {
+				dErrs[i] = dErr
+			}
+		}(index, object)
+	}
+	wg.Wait()
+
+	// Collect deleted objects and errors if any.
+	var deletedObjects []ObjectIdentifier
+	var deleteErrors []DeleteError
+	for index, err := range dErrs {
+		object := deleteObjects.Objects[index]
+		// Success deleted objects are collected separately.
+		if err == nil {
+			deletedObjects = append(deletedObjects, object)
+			continue
+		}
+		if _, ok := errorCause(err).(ObjectNotFound); ok {
+			// If the object is not found it should be
+			// accounted as deleted as per S3 spec.
+			deletedObjects = append(deletedObjects, object)
+			continue
+		}
+		errorIf(err, "Unable to delete object. %s", object.ObjectName)
+		// Error during delete should be collected separately.
+		deleteErrors = append(deleteErrors, DeleteError{
+			Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
+			Message: errorCodeResponse[toAPIErrorCode(err)].Description,
+			Key:     object.ObjectName,
+		})
+	}
+
+	// Generate response
+	response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
+	encodedSuccessResponse := encodeResponse(response)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+
+	// Notify deleted event for objects.
+	for _, dobj := range deletedObjects {
+		eventNotify(eventData{
+			Type:   ObjectRemovedDelete,
+			Bucket: bucket,
+			ObjInfo: ObjectInfo{
+				Name: dobj.ObjectName,
+			},
+			ReqParams: map[string]string{
+				"sourceIPAddress": r.RemoteAddr,
+			},
+		})
+	}
+}
+
+// PutBucketHandler - PUT Bucket
+// ----------
+// This implementation of the PUT operation creates a new bucket for authenticated request
+func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	// PutBucket does not have any bucket action.
+	if s3Error := checkRequestAuthType(r, "", "", "us-east-1"); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Validate if incoming location constraint is valid, reject
+	// requests which do not follow valid region requirements.
+	if s3Error := isValidLocationConstraint(r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	bucketLock := globalNSMutex.NewNSLock(bucket, "")
+	bucketLock.Lock()
+	defer bucketLock.Unlock()
+
+	// Proceed to creating a bucket.
+	err := objectAPI.MakeBucket(bucket)
+	if err != nil {
+		errorIf(err, "Unable to create a bucket.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// Make sure to add Location information here only for bucket
+	w.Header().Set("Location", getLocation(r))
+	writeSuccessResponse(w, nil)
+}
+
+// PostPolicyBucketHandler - POST policy
+// ----------
+// This implementation of the POST operation handles object creation with a specified
+// signature policy in multipart/form-data
+func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	// Here the parameter is the size of the form data that should
+	// be loaded in memory, the remaining being put in temporary files.
+	reader, err := r.MultipartReader()
+	if err != nil {
+		errorIf(err, "Unable to initialize multipart reader.")
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
+		return
+	}
+
+	fileBody, fileName, formValues, err := extractPostPolicyFormValues(reader)
+	if err != nil {
+		errorIf(err, "Unable to parse form values.")
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
+		return
+	}
+	bucket := mux.Vars(r)["bucket"]
+	formValues["Bucket"] = bucket
+
+	if fileName != "" && strings.Contains(formValues["Key"], "${filename}") {
+		// S3 feature to replace ${filename} found in Key form field
+		// by the filename attribute passed in multipart
+		formValues["Key"] = strings.Replace(formValues["Key"], "${filename}", fileName, -1)
+	}
+	object := formValues["Key"]
+
+	// Verify policy signature.
+	apiErr := doesPolicySignatureMatch(formValues)
+	if apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+
+	policyBytes, err := base64.StdEncoding.DecodeString(formValues["Policy"])
+	if err != nil {
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
+		return
+	}
+
+	postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
+	if err != nil {
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
+		return
+	}
+
+	// Make sure formValues adhere to policy restrictions.
+	if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+
+	// Use rangeReader to ensure that object size is within expected range.
+	lengthRange := postPolicyForm.Conditions.ContentLengthRange
+	if lengthRange.Valid {
+		// If policy restricted the size of the object.
+		fileBody = &rangeReader{
+			Reader: fileBody,
+			Min:    lengthRange.Min,
+			Max:    lengthRange.Max,
+		}
+	} else {
+		// Default values of min/max size of the object.
+		fileBody = &rangeReader{
+			Reader: fileBody,
+			Min:    0,
+			Max:    maxObjectSize,
+		}
+	}
+
+	// Save metadata.
+	metadata := make(map[string]string)
+	// Nothing to store right now.
+
+	sha256sum := ""
+
+	objectLock := globalNSMutex.NewNSLock(bucket, object)
+	objectLock.Lock()
+	defer objectLock.Unlock()
+
+	objInfo, err := objectAPI.PutObject(bucket, object, -1, fileBody, metadata, sha256sum)
+	if err != nil {
+		errorIf(err, "Unable to create object.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+	w.Header().Set("Location", getObjectLocation(bucket, object))
+
+	// Set common headers.
+	setCommonHeaders(w)
+
+	// Write successful response.
+	writeSuccessNoContent(w)
+
+	// Notify object created event.
+	eventNotify(eventData{
+		Type:    ObjectCreatedPost,
+		Bucket:  bucket,
+		ObjInfo: objInfo,
+		ReqParams: map[string]string{
+			"sourceIPAddress": r.RemoteAddr,
+		},
+	})
+}
+
+// HeadBucketHandler - HEAD Bucket
+// ----------
+// This operation is useful to determine if a bucket exists.
+// The operation returns a 200 OK if the bucket exists and you
+// have permission to access it. Otherwise, the operation might
+// return responses such as 404 Not Found and 403 Forbidden.
+func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	bucketLock := globalNSMutex.NewNSLock(bucket, "")
+	bucketLock.RLock()
+	defer bucketLock.RUnlock()
+
+	if _, err := objectAPI.GetBucketInfo(bucket); err != nil {
+		errorIf(err, "Unable to fetch bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// DeleteBucketHandler - Delete bucket
+func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	// DeleteBucket does not have any bucket action.
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	bucketLock := globalNSMutex.NewNSLock(bucket, "")
+	bucketLock.Lock()
+	defer bucketLock.Unlock()
+
+	// Attempt to delete bucket.
+	if err := objectAPI.DeleteBucket(bucket); err != nil {
+		errorIf(err, "Unable to delete a bucket.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Delete bucket access policy, if present - ignore any errors.
+	_ = removeBucketPolicy(bucket, objectAPI)
+
+	// Delete notification config, if present - ignore any errors.
+	_ = removeNotificationConfig(bucket, objectAPI)
+
+	// Delete listener config, if present - ignore any errors.
+	_ = removeListenerConfig(bucket, objectAPI)
+
+	// Write success response.
+	writeSuccessNoContent(w)
+}

cmd/bucket-handlers_test.go

@@ -0,0 +1,773 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/xml"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+)
+
+// Wrapper for calling GetBucketPolicy HTTP handler tests for both XL multiple disks and single node setup.
+func TestGetBucketLocationHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testGetBucketLocationHandler, []string{"GetBucketLocation"})
+}
+
+func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	initBucketPolicies(obj)
+
+	// test cases with sample input and expected output.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+		locationResponse   []byte
+		errorResponse      APIErrorResponse
+		shouldPass         bool
+	}{
+		// Test case - 1.
+		// Tests for authenticated request and proper response.
+		{
+			bucketName:         bucketName,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusOK,
+			locationResponse: []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></LocationConstraint>`),
+			errorResponse: APIErrorResponse{},
+			shouldPass:    true,
+		},
+		// Test case - 2.
+		// Tests for signature mismatch error.
+		{
+			bucketName:         bucketName,
+			accessKey:          "abcd",
+			secretKey:          "abcd",
+			expectedRespStatus: http.StatusForbidden,
+			locationResponse:   []byte(""),
+			errorResponse: APIErrorResponse{
+				Resource: "/" + bucketName + "/",
+				Code:     "InvalidAccessKeyID",
+				Message:  "The access key ID you provided does not exist in our records.",
+			},
+			shouldPass: false,
+		},
+	}
+
+	for i, testCase := range testCases {
+		if i != 1 {
+			continue
+		}
+		// initialize httptest Recorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for Get bucket location.
+		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+		if !bytes.Equal(testCase.locationResponse, rec.Body.Bytes()) && testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.locationResponse), string(rec.Body.Bytes()))
+		}
+		errorResponse := APIErrorResponse{}
+		err = xml.Unmarshal(rec.Body.Bytes(), &errorResponse)
+		if err != nil && !testCase.shouldPass {
+			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, string(rec.Body.Bytes()))
+		}
+		if errorResponse.Resource != testCase.errorResponse.Resource {
+			t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource)
+		}
+		if errorResponse.Message != testCase.errorResponse.Message {
+			t.Errorf("Test %d: %s: Expected the error message to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Message, errorResponse.Message)
+		}
+		if errorResponse.Code != testCase.errorResponse.Code {
+			t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code)
+		}
+
+		// Verify response of the V2 signed HTTP request.
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV2 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV2, reqV2)
+		if recV2.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
+		}
+
+		errorResponse = APIErrorResponse{}
+		err = xml.Unmarshal(recV2.Body.Bytes(), &errorResponse)
+		if err != nil && !testCase.shouldPass {
+			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, string(recV2.Body.Bytes()))
+		}
+		if errorResponse.Resource != testCase.errorResponse.Resource {
+			t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource)
+		}
+		if errorResponse.Message != testCase.errorResponse.Message {
+			t.Errorf("Test %d: %s: Expected the error message to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Message, errorResponse.Message)
+		}
+		if errorResponse.Code != testCase.errorResponse.Code {
+			t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code)
+		}
+
+	}
+
+	// Test for Anonymous/unsigned http request.
+	// ListBucketsHandler doesn't support bucket policies, setting the policies shouldn't make any difference.
+	anonReq, err := newTestRequest("GET", getBucketLocationURL("", bucketName), 0, nil)
+	if err != nil {
+		t.Fatalf("Minio %s: Failed to create an anonymous request.", instanceType)
+	}
+
+	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
+	// sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the
+	// unsigned request goes through and its validated again.
+	ExecObjectLayerAPIAnonTest(t, "TestGetBucketLocationHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyBucketStatement)
+
+	// HTTP request for testing when `objectLayer` is set to `nil`.
+	// There is no need to use an existing bucket and valid input for creating the request
+	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
+	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
+
+	nilBucket := "dummy-bucket"
+	nilReq, err := newTestRequest("GET", getBucketLocationURL("", nilBucket), 0, nil)
+
+	if err != nil {
+		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
+	}
+	// Executes the object layer set to `nil` test.
+	// `ExecObjectLayerAPINilTest` manages the operation.
+	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
+}
+
+// Wrapper for calling HeadBucket HTTP handler tests for both XL multiple disks and single node setup.
+func TestHeadBucketHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testHeadBucketHandler, []string{"HeadBucket"})
+}
+
+func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	initBucketPolicies(obj)
+
+	// test cases with sample input and expected output.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		// Test case - 1.
+		// Bucket exists.
+		{
+			bucketName:         bucketName,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusOK,
+		},
+		// Test case - 2.
+		// Non-existent bucket name.
+		{
+			bucketName:         "2333",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 3.
+		// Testing for signature mismatch error.
+		// setting invalid acess and secret key.
+		{
+			bucketName:         bucketName,
+			accessKey:          "abcd",
+			secretKey:          "abcd",
+			expectedRespStatus: http.StatusForbidden,
+		},
+	}
+
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for HEAD bucket.
+		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+
+		// Verify response the V2 signed HTTP request.
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV2 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV2, reqV2)
+		if recV2.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
+		}
+
+	}
+
+	// Test for Anonymous/unsigned http request.
+	anonReq, err := newTestRequest("HEAD", getHEADBucketURL("", bucketName), 0, nil)
+
+	if err != nil {
+		t.Fatalf("Minio %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
+			instanceType, bucketName, err)
+	}
+
+	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
+	// sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the
+	// unsigned request goes through and its validated again.
+	ExecObjectLayerAPIAnonTest(t, "TestHeadBucketHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyBucketStatement)
+
+	// HTTP request for testing when `objectLayer` is set to `nil`.
+	// There is no need to use an existing bucket and valid input for creating the request
+	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
+	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
+
+	nilBucket := "dummy-bucket"
+	nilReq, err := newTestRequest("HEAD", getHEADBucketURL("", nilBucket), 0, nil)
+
+	if err != nil {
+		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
+	}
+	// execute the object layer set to `nil` test.
+	// `ExecObjectLayerAPINilTest` manages the operation.
+	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
+}
+
+// Wrapper for calling TestListMultipartUploadsHandler tests for both XL multiple disks and single node setup.
+func TestListMultipartUploadsHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testListMultipartUploadsHandler, []string{"ListMultipartUploads"})
+}
+
+// testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
+func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	initBucketPolicies(obj)
+
+	// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
+	// and success responses.
+	testCases := []struct {
+		// Inputs to ListMultipartUploads.
+		bucket             string
+		prefix             string
+		keyMarker          string
+		uploadIDMarker     string
+		delimiter          string
+		maxUploads         string
+		accessKey          string
+		secretKey          string
+		expectedRespStatus int
+		shouldPass         bool
+	}{
+		// Test case - 1.
+		// Setting invalid bucket name.
+		{
+			bucket:             ".test",
+			prefix:             "",
+			keyMarker:          "",
+			uploadIDMarker:     "",
+			delimiter:          "",
+			maxUploads:         "0",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusBadRequest,
+			shouldPass:         false,
+		},
+		// Test case - 2.
+		// Setting a non-existent bucket.
+		{
+			bucket:             "volatile-bucket-1",
+			prefix:             "",
+			keyMarker:          "",
+			uploadIDMarker:     "",
+			delimiter:          "",
+			maxUploads:         "0",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNotFound,
+			shouldPass:         false,
+		},
+		// Test case -3.
+		// Setting invalid delimiter, expecting the HTTP response status to be http.StatusNotImplemented.
+		{
+			bucket:             bucketName,
+			prefix:             "",
+			keyMarker:          "",
+			uploadIDMarker:     "",
+			delimiter:          "-",
+			maxUploads:         "0",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNotImplemented,
+			shouldPass:         false,
+		},
+		// Test case - 4.
+		// Setting Invalid prefix and marker combination.
+		{
+			bucket:             bucketName,
+			prefix:             "asia",
+			keyMarker:          "europe-object",
+			uploadIDMarker:     "",
+			delimiter:          "",
+			maxUploads:         "0",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNotImplemented,
+			shouldPass:         false,
+		},
+		// Test case - 5.
+		// Invalid upload id and marker combination.
+		{
+			bucket:             bucketName,
+			prefix:             "asia",
+			keyMarker:          "asia/europe/",
+			uploadIDMarker:     "abc",
+			delimiter:          "",
+			maxUploads:         "0",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNotImplemented,
+			shouldPass:         false,
+		},
+		// Test case - 6.
+		// Setting a negative value to max-uploads paramater, should result in http.StatusBadRequest.
+		{
+			bucket:             bucketName,
+			prefix:             "",
+			keyMarker:          "",
+			uploadIDMarker:     "",
+			delimiter:          "",
+			maxUploads:         "-1",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusBadRequest,
+			shouldPass:         false,
+		},
+		// Test case - 7.
+		// Case with right set of parameters,
+		// should result in success 200OK.
+		{
+			bucket:             bucketName,
+			prefix:             "",
+			keyMarker:          "",
+			uploadIDMarker:     "",
+			delimiter:          "/",
+			maxUploads:         "100",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusOK,
+			shouldPass:         true,
+		},
+		// Test case - 8.
+		// Good case without delimiter.
+		{
+			bucket:             bucketName,
+			prefix:             "",
+			keyMarker:          "",
+			uploadIDMarker:     "",
+			delimiter:          "",
+			maxUploads:         "100",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusOK,
+			shouldPass:         true,
+		},
+		// Test case - 9.
+		// Setting Invalid AccessKey and SecretKey to induce and verify Signature Mismatch error.
+		{
+			bucket:             bucketName,
+			prefix:             "",
+			keyMarker:          "",
+			uploadIDMarker:     "",
+			delimiter:          "",
+			maxUploads:         "100",
+			accessKey:          "abcd",
+			secretKey:          "abcd",
+			expectedRespStatus: http.StatusForbidden,
+			shouldPass:         true,
+		},
+	}
+
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+
+		// construct HTTP request for List multipart uploads endpoint.
+		u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads)
+		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+		if gerr != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", i+1, instanceType, gerr)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+
+		// Verify response the V2 signed HTTP request.
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV2 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+
+		// verify response for V2 signed HTTP request.
+		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV2, reqV2)
+		if recV2.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
+		}
+	}
+
+	// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+	rec := httptest.NewRecorder()
+
+	// construct HTTP request for List multipart uploads endpoint.
+	u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "")
+	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "") // Generate an anonymous request.
+	if err != nil {
+		t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", instanceType, err)
+	}
+	// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+	// Call the ServeHTTP to execute the handler.
+	apiRouter.ServeHTTP(rec, req)
+	if rec.Code != http.StatusForbidden {
+		t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code)
+	}
+
+	url := getListMultipartUploadsURLWithParams("", testCases[6].bucket, testCases[6].prefix, testCases[6].keyMarker,
+		testCases[6].uploadIDMarker, testCases[6].delimiter, testCases[6].maxUploads)
+	// Test for Anonymous/unsigned http request.
+	anonReq, err := newTestRequest("GET", url, 0, nil)
+	if err != nil {
+		t.Fatalf("Minio %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
+			instanceType, bucketName, err)
+	}
+
+	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
+	// sets the bucket policy using the policy statement generated from `getWriteOnlyBucketStatement` so that the
+	// unsigned request goes through and its validated again.
+	ExecObjectLayerAPIAnonTest(t, "TestListMultipartUploadsHandler", bucketName, "", instanceType, apiRouter, anonReq, getWriteOnlyBucketStatement)
+
+	// HTTP request for testing when `objectLayer` is set to `nil`.
+	// There is no need to use an existing bucket and valid input for creating the request
+	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
+	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
+
+	nilBucket := "dummy-bucket"
+	url = getListMultipartUploadsURLWithParams("", nilBucket, "dummy-prefix", testCases[6].keyMarker,
+		testCases[6].uploadIDMarker, testCases[6].delimiter, testCases[6].maxUploads)
+
+	nilReq, err := newTestRequest("GET", url, 0, nil)
+
+	if err != nil {
+		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
+	}
+	// execute the object layer set to `nil` test.
+	// `ExecObjectLayerAPINilTest` manages the operation.
+	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
+}
+
+// Wrapper for calling TestListBucketsHandler tests for both XL multiple disks and single node setup.
+func TestListBucketsHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testListBucketsHandler, []string{"ListBuckets"})
+}
+
+// testListBucketsHandler - Tests validate listing of buckets.
+func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+
+	testCases := []struct {
+		bucketName         string
+		accessKey          string
+		secretKey          string
+		expectedRespStatus int
+	}{
+		// Test case - 1.
+		// Validate a good case request succeeds.
+		{
+			bucketName:         bucketName,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusOK,
+		},
+		// Test case - 2.
+		// Test case with invalid accessKey to produce and validate Signature MisMatch error.
+		{
+			bucketName:         bucketName,
+			accessKey:          "abcd",
+			secretKey:          "abcd",
+			expectedRespStatus: http.StatusForbidden,
+		},
+	}
+
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		if lerr != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: <ERROR> %v", i+1, instanceType, lerr)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+
+		// Verify response of the V2 signed HTTP request.
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV2 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+
+		// verify response for V2 signed HTTP request.
+		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV2, reqV2)
+		if recV2.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
+		}
+	}
+
+	// Test for Anonymous/unsigned http request.
+	// ListBucketsHandler doesn't support bucket policies, setting the policies shouldn't make a difference.
+	anonReq, err := newTestRequest("GET", getListBucketURL(""), 0, nil)
+
+	if err != nil {
+		t.Fatalf("Minio %s: Failed to create an anonymous request.", instanceType)
+	}
+
+	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
+	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
+	// unsigned request goes through and its validated again.
+	ExecObjectLayerAPIAnonTest(t, "ListBucketsHandler", "", "", instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+
+	// HTTP request for testing when `objectLayer` is set to `nil`.
+	// There is no need to use an existing bucket and valid input for creating the request
+	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
+	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
+
+	nilReq, err := newTestRequest("GET", getListBucketURL(""), 0, nil)
+
+	if err != nil {
+		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
+	}
+	// execute the object layer set to `nil` test.
+	// `ExecObjectLayerAPINilTest` manages the operation.
+	ExecObjectLayerAPINilTest(t, "", "", instanceType, apiRouter, nilReq)
+}
+
+// Wrapper for calling DeleteMultipleObjects HTTP handler tests for both XL multiple disks and single node setup.
+func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testAPIDeleteMultipleObjectsHandler, []string{"DeleteMultipleObjects"})
+}
+
+func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	initBucketPolicies(obj)
+
+	var err error
+	// register event notifier.
+	err = initEventNotifier(obj)
+	if err != nil {
+		t.Fatal("Notifier initialization failed.")
+	}
+
+	contentBytes := []byte("hello")
+	sha256sum := ""
+	var objectNames []string
+	for i := 0; i < 10; i++ {
+		objectName := "test-object-" + strconv.Itoa(i)
+		// uploading the object.
+		_, err = obj.PutObject(bucketName, objectName, int64(len(contentBytes)), bytes.NewBuffer(contentBytes),
+			make(map[string]string), sha256sum)
+		// if object upload fails stop the test.
+		if err != nil {
+			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
+		}
+
+		// object used for the test.
+		objectNames = append(objectNames, objectName)
+	}
+
+	getObjectIdentifierList := func(objectNames []string) (objectIdentifierList []ObjectIdentifier) {
+		for _, objectName := range objectNames {
+			objectIdentifierList = append(objectIdentifierList, ObjectIdentifier{objectName})
+		}
+
+		return objectIdentifierList
+	}
+
+	requestList := []DeleteObjectsRequest{
+		{Quiet: false, Objects: getObjectIdentifierList(objectNames[:5])},
+		{Quiet: true, Objects: getObjectIdentifierList(objectNames[5:])},
+	}
+
+	// generate multi objects delete response.
+	successRequest0 := encodeResponse(requestList[0])
+	successResponse0 := generateMultiDeleteResponse(requestList[0].Quiet, requestList[0].Objects, nil)
+	encodedSuccessResponse0 := encodeResponse(successResponse0)
+
+	successRequest1 := encodeResponse(requestList[1])
+	successResponse1 := generateMultiDeleteResponse(requestList[1].Quiet, requestList[1].Objects, nil)
+	encodedSuccessResponse1 := encodeResponse(successResponse1)
+
+	// generate multi objects delete response for errors.
+	// errorRequest := encodeResponse(requestList[1])
+	errorResponse := generateMultiDeleteResponse(requestList[1].Quiet, requestList[1].Objects, nil)
+	encodedErrorResponse := encodeResponse(errorResponse)
+
+	testCases := []struct {
+		bucket             string
+		objects            []byte
+		accessKey          string
+		secretKey          string
+		expectedContent    []byte
+		expectedRespStatus int
+	}{
+		// Test case - 1.
+		// Delete objects with invalid access key.
+		{
+			bucket:             bucketName,
+			objects:            successRequest0,
+			accessKey:          "Invalid-AccessID",
+			secretKey:          credentials.SecretAccessKey,
+			expectedContent:    nil,
+			expectedRespStatus: http.StatusForbidden,
+		},
+		// Test case - 2.
+		// Delete valid objects with quiet flag off.
+		{
+			bucket:             bucketName,
+			objects:            successRequest0,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedContent:    encodedSuccessResponse0,
+			expectedRespStatus: http.StatusOK,
+		},
+		// Test case - 3.
+		// Delete valid objects with quiet flag on.
+		{
+			bucket:             bucketName,
+			objects:            successRequest1,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedContent:    encodedSuccessResponse1,
+			expectedRespStatus: http.StatusOK,
+		},
+		// Test case - 4.
+		// Delete previously deleted objects.
+		{
+			bucket:             bucketName,
+			objects:            successRequest1,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedContent:    encodedErrorResponse,
+			expectedRespStatus: http.StatusOK,
+		},
+	}
+
+	for i, testCase := range testCases {
+		var req *http.Request
+		var actualContent []byte
+
+		// Indicating that all parts are uploaded and initiating completeMultipartUpload.
+		req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
+			int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Failed to create HTTP request for DeleteMultipleObjects: <ERROR> %v", err)
+		}
+
+		rec := httptest.NewRecorder()
+
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to executes the registered handler.
+		apiRouter.ServeHTTP(rec, req)
+		// Assert the response code with the expected status.
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Case %d: Minio %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+
+		// read the response body.
+		actualContent, err = ioutil.ReadAll(rec.Body)
+		if err != nil {
+			t.Fatalf("Test %d : Minio %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
+		}
+
+		// Verify whether the bucket obtained object is same as the one created.
+		if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) {
+			t.Errorf("Test %d : Minio %s: Object content differs from expected value.", i+1, instanceType)
+		}
+	}
+
+	// Currently anonymous user cannot delete multiple objects in Minio server, hence no test case is required.
+
+	// HTTP request to test the case of `objectLayer` being set to `nil`.
+	// There is no need to use an existing bucket or valid input for creating the request,
+	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
+	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
+	// Indicating that all parts are uploaded and initiating completeMultipartUpload.
+	nilBucket := "dummy-bucket"
+	nilObject := ""
+
+	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "")
+	if err != nil {
+		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
+	}
+	// execute the object layer set to `nil` test.
+	// `ExecObjectLayerAPINilTest` manages the operation.
+	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
+}

cmd/bucket-metadata.go

@@ -0,0 +1,173 @@
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"net/rpc"
+)
+
+// BucketMetaState - Interface to update bucket metadata in-memory
+// state.
+type BucketMetaState interface {
+	// Updates bucket notification
+	UpdateBucketNotification(args *SetBucketNotificationPeerArgs) error
+
+	// Updates bucket listener
+	UpdateBucketListener(args *SetBucketListenerPeerArgs) error
+
+	// Updates bucket policy
+	UpdateBucketPolicy(args *SetBucketPolicyPeerArgs) error
+
+	// Sends event
+	SendEvent(args *EventArgs) error
+}
+
+// BucketUpdater - Interface implementer calls one of BucketMetaState's methods.
+type BucketUpdater interface {
+	BucketUpdate(client BucketMetaState) error
+}
+
+// Type that implements BucketMetaState for local node.
+type localBucketMetaState struct {
+	ObjectAPI func() ObjectLayer
+}
+
+// localBucketMetaState.UpdateBucketNotification - updates in-memory global bucket
+// notification info.
+func (lc *localBucketMetaState) UpdateBucketNotification(args *SetBucketNotificationPeerArgs) error {
+	// check if object layer is available.
+	objAPI := lc.ObjectAPI()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	globalEventNotifier.SetBucketNotificationConfig(args.Bucket, args.NCfg)
+
+	return nil
+}
+
+// localBucketMetaState.UpdateBucketListener - updates in-memory global bucket
+// listeners info.
+func (lc *localBucketMetaState) UpdateBucketListener(args *SetBucketListenerPeerArgs) error {
+	// check if object layer is available.
+	objAPI := lc.ObjectAPI()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// Update in-memory notification config.
+	return globalEventNotifier.SetBucketListenerConfig(args.Bucket, args.LCfg)
+}
+
+// localBucketMetaState.UpdateBucketPolicy - updates in-memory global bucket
+// policy info.
+func (lc *localBucketMetaState) UpdateBucketPolicy(args *SetBucketPolicyPeerArgs) error {
+	// check if object layer is available.
+	objAPI := lc.ObjectAPI()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	var pCh policyChange
+	if err := json.Unmarshal(args.PChBytes, &pCh); err != nil {
+		return err
+	}
+
+	return globalBucketPolicies.SetBucketPolicy(args.Bucket, pCh)
+}
+
+// localBucketMetaState.SendEvent - sends event to local event notifier via
+// `globalEventNotifier`
+func (lc *localBucketMetaState) SendEvent(args *EventArgs) error {
+	// check if object layer is available.
+	objAPI := lc.ObjectAPI()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	return globalEventNotifier.SendListenerEvent(args.Arn, args.Event)
+}
+
+// Type that implements BucketMetaState for remote node.
+type remoteBucketMetaState struct {
+	*AuthRPCClient
+}
+
+// remoteBucketMetaState.UpdateBucketNotification - sends bucket notification
+// change to remote peer via RPC call.
+func (rc *remoteBucketMetaState) UpdateBucketNotification(args *SetBucketNotificationPeerArgs) error {
+	reply := GenericReply{}
+	err := rc.Call("S3.SetBucketNotificationPeer", args, &reply)
+	// Check for network error and retry once.
+	if err != nil && err == rpc.ErrShutdown {
+		// Close the underlying connection to attempt once more.
+		rc.Close()
+
+		// Attempt again and proceed.
+		err = rc.Call("S3.SetBucketNotificationPeer", args, &reply)
+	}
+	return err
+}
+
+// remoteBucketMetaState.UpdateBucketListener - sends bucket listener change to
+// remote peer via RPC call.
+func (rc *remoteBucketMetaState) UpdateBucketListener(args *SetBucketListenerPeerArgs) error {
+	reply := GenericReply{}
+	err := rc.Call("S3.SetBucketListenerPeer", args, &reply)
+	// Check for network error and retry once.
+	if err != nil && err == rpc.ErrShutdown {
+		// Close the underlying connection to attempt once more.
+		rc.Close()
+
+		// Attempt again and proceed.
+		err = rc.Call("S3.SetBucketListenerPeer", args, &reply)
+	}
+	return err
+}
+
+// remoteBucketMetaState.UpdateBucketPolicy - sends bucket policy change to remote
+// peer via RPC call.
+func (rc *remoteBucketMetaState) UpdateBucketPolicy(args *SetBucketPolicyPeerArgs) error {
+	reply := GenericReply{}
+	err := rc.Call("S3.SetBucketPolicyPeer", args, &reply)
+	// Check for network error and retry once.
+	if err != nil && err == rpc.ErrShutdown {
+		// Close the underlying connection to attempt once more.
+		rc.Close()
+
+		// Attempt again and proceed.
+		err = rc.Call("S3.SetBucketPolicyPeer", args, &reply)
+	}
+	return err
+}
+
+// remoteBucketMetaState.SendEvent - sends event for bucket listener to remote
+// peer via RPC call.
+func (rc *remoteBucketMetaState) SendEvent(args *EventArgs) error {
+	reply := GenericReply{}
+	err := rc.Call("S3.Event", args, &reply)
+	// Check for network error and retry once.
+	if err != nil && err == rpc.ErrShutdown {
+		// Close the underlying connection to attempt once more.
+		rc.Close()
+
+		// Attempt again and proceed.
+		err = rc.Call("S3.Event", args, &reply)
+	}
+	return err
+}

cmd/bucket-notification-datatypes.go

@@ -0,0 +1,172 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"errors"
+)
+
+// Represents the criteria for the filter rule.
+type filterRule struct {
+	Name  string `xml:"Name"`
+	Value string `xml:"Value"`
+}
+
+// Collection of filter rules per service config.
+type keyFilter struct {
+	FilterRules []filterRule `xml:"FilterRule,omitempty"`
+}
+
+type filterStruct struct {
+	Key keyFilter `xml:"S3Key,omitempty" json:"S3Key,omitempty"`
+}
+
+// ServiceConfig - Common elements of service notification.
+type ServiceConfig struct {
+	Events []string     `xml:"Event" json:"Event"`
+	Filter filterStruct `xml:"Filter" json:"Filter"`
+	ID     string       `xml:"Id" json:"Id"`
+}
+
+// Queue SQS configuration.
+type queueConfig struct {
+	ServiceConfig
+	QueueARN string `xml:"Queue"`
+}
+
+// Topic SNS configuration, this is a compliance field not used by minio yet.
+type topicConfig struct {
+	ServiceConfig
+	TopicARN string `xml:"Topic" json:"Topic"`
+}
+
+// Lambda function configuration, this is a compliance field not used by minio yet.
+type lambdaConfig struct {
+	ServiceConfig
+	LambdaARN string `xml:"CloudFunction"`
+}
+
+// Notification configuration structure represents the XML format of
+// notification configuration of buckets.
+type notificationConfig struct {
+	XMLName       xml.Name       `xml:"NotificationConfiguration"`
+	QueueConfigs  []queueConfig  `xml:"QueueConfiguration"`
+	LambdaConfigs []lambdaConfig `xml:"CloudFunctionConfiguration"`
+}
+
+// listenerConfig structure represents run-time notification
+// configuration for live listeners
+type listenerConfig struct {
+	TopicConfig  topicConfig `json:"TopicConfiguration"`
+	TargetServer string      `json:"TargetServer"`
+}
+
+// Internal error used to signal notifications not set.
+var errNoSuchNotifications = errors.New("The specified bucket does not have bucket notifications")
+
+// EventName is AWS S3 event type:
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html
+type EventName int
+
+const (
+	// ObjectCreatedPut is s3:ObjectCreated:Put
+	ObjectCreatedPut EventName = iota
+	// ObjectCreatedPost is s3:ObjectCreated:Post
+	ObjectCreatedPost
+	// ObjectCreatedCopy is s3:ObjectCreated:Copy
+	ObjectCreatedCopy
+	// ObjectCreatedCompleteMultipartUpload is s3:ObjectCreated:CompleteMultipartUpload
+	ObjectCreatedCompleteMultipartUpload
+	// ObjectRemovedDelete is s3:ObjectRemoved:Delete
+	ObjectRemovedDelete
+)
+
+// Stringer interface for event name.
+func (eventName EventName) String() string {
+	switch eventName {
+	case ObjectCreatedPut:
+		return "s3:ObjectCreated:Put"
+	case ObjectCreatedPost:
+		return "s3:ObjectCreated:Post"
+	case ObjectCreatedCopy:
+		return "s3:ObjectCreated:Copy"
+	case ObjectCreatedCompleteMultipartUpload:
+		return "s3:ObjectCreated:CompleteMultipartUpload"
+	case ObjectRemovedDelete:
+		return "s3:ObjectRemoved:Delete"
+	default:
+		return "s3:Unknown"
+	}
+}
+
+// Indentity represents the user id, this is a compliance field.
+type identity struct {
+	PrincipalID string `json:"principalId"`
+}
+
+func defaultIdentity() identity {
+	return identity{"minio"}
+}
+
+// Notification event bucket metadata.
+type bucketMeta struct {
+	Name          string   `json:"name"`
+	OwnerIdentity identity `json:"ownerIdentity"`
+	ARN           string   `json:"arn"`
+}
+
+// Notification event object metadata.
+type objectMeta struct {
+	Key       string `json:"key"`
+	Size      int64  `json:"size,omitempty"`
+	ETag      string `json:"eTag,omitempty"`
+	VersionID string `json:"versionId,omitempty"`
+	Sequencer string `json:"sequencer"`
+}
+
+// Notification event server specific metadata.
+type eventMeta struct {
+	SchemaVersion   string     `json:"s3SchemaVersion"`
+	ConfigurationID string     `json:"configurationId"`
+	Bucket          bucketMeta `json:"bucket"`
+	Object          objectMeta `json:"object"`
+}
+
+// NotificationEvent represents an Amazon an S3 bucket notification event.
+type NotificationEvent struct {
+	EventVersion      string            `json:"eventVersion"`
+	EventSource       string            `json:"eventSource"`
+	AwsRegion         string            `json:"awsRegion"`
+	EventTime         string            `json:"eventTime"`
+	EventName         string            `json:"eventName"`
+	UserIdentity      identity          `json:"userIdentity"`
+	RequestParameters map[string]string `json:"requestParameters"`
+	ResponseElements  map[string]string `json:"responseElements"`
+	S3                eventMeta         `json:"s3"`
+}
+
+// Represents the minio sqs type and account id's.
+type arnSQS struct {
+	Type      string
+	AccountID string
+}
+
+// Stringer for constructing AWS ARN compatible string.
+func (m arnSQS) String() string {
+	return minioSqs + serverConfig.GetRegion() + ":" + m.AccountID + ":" + m.Type
+}

cmd/bucket-notification-handlers.go

@@ -0,0 +1,442 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+)
+
+const (
+	bucketConfigPrefix       = "buckets"
+	bucketNotificationConfig = "notification.xml"
+	bucketListenerConfig     = "listener.json"
+)
+
+// GetBucketNotificationHandler - This implementation of the GET
+// operation uses the notification subresource to return the
+// notification configuration of a bucket. If notifications are
+// not enabled on the bucket, the operation returns an empty
+// NotificationConfiguration element.
+func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	_, err := objAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err, "Unable to find bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Attempt to successfully load notification config.
+	nConfig, err := loadNotificationConfig(bucket, objAPI)
+	if err != nil && err != errNoSuchNotifications {
+		errorIf(err, "Unable to read notification configuration.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// For no notifications we write a dummy XML.
+	if err == errNoSuchNotifications {
+		// Complies with the s3 behavior in this regard.
+		nConfig = &notificationConfig{}
+	}
+	notificationBytes, err := xml.Marshal(nConfig)
+	if err != nil {
+		// For any marshalling failure.
+		errorIf(err, "Unable to marshal notification configuration into XML.", err)
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// Success.
+	writeSuccessResponse(w, notificationBytes)
+}
+
+// PutBucketNotificationHandler - Minio notification feature enables
+// you to receive notifications when certain events happen in your bucket.
+// Using this API, you can replace an existing notification configuration.
+// The configuration is an XML file that defines the event types that you
+// want Minio to publish and the destination where you want Minio to publish
+// an event notification when it detects an event of the specified type.
+// By default, your bucket has no event notifications configured. That is,
+// the notification configuration will be an empty NotificationConfiguration.
+func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	_, err := objectAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err, "Unable to find bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// If Content-Length is unknown or zero, deny the request. PutBucketNotification
+	// always needs a Content-Length if incoming request is not chunked.
+	if !contains(r.TransferEncoding, "chunked") {
+		if r.ContentLength == -1 {
+			writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+			return
+		}
+	}
+
+	// Reads the incoming notification configuration.
+	var buffer bytes.Buffer
+	if r.ContentLength >= 0 {
+		_, err = io.CopyN(&buffer, r.Body, r.ContentLength)
+	} else {
+		_, err = io.Copy(&buffer, r.Body)
+	}
+	if err != nil {
+		errorIf(err, "Unable to read incoming body.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	var notificationCfg notificationConfig
+	// Unmarshal notification bytes.
+	notificationConfigBytes := buffer.Bytes()
+	if err = xml.Unmarshal(notificationConfigBytes, &notificationCfg); err != nil {
+		errorIf(err, "Unable to parse notification configuration XML.")
+		writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
+		return
+	} // Successfully marshalled notification configuration.
+
+	// Validate unmarshalled bucket notification configuration.
+	if s3Error := validateNotificationConfig(notificationCfg); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Put bucket notification config.
+	err = PutBucketNotificationConfig(bucket, &notificationCfg, objectAPI)
+	if err != nil {
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Success.
+	writeSuccessResponse(w, nil)
+}
+
+// PutBucketNotificationConfig - Put a new notification config for a
+// bucket (overwrites any previous config) persistently, updates
+// global in-memory state, and notify other nodes in the cluster (if
+// any)
+func PutBucketNotificationConfig(bucket string, ncfg *notificationConfig, objAPI ObjectLayer) error {
+	if ncfg == nil {
+		return errInvalidArgument
+	}
+
+	// Acquire a write lock on bucket before modifying its
+	// configuration.
+	bucketLock := globalNSMutex.NewNSLock(bucket, "")
+	bucketLock.Lock()
+	// Release lock after notifying peers
+	defer bucketLock.Unlock()
+
+	// persist config to disk
+	err := persistNotificationConfig(bucket, ncfg, objAPI)
+	if err != nil {
+		return fmt.Errorf("Unable to persist Bucket notification config to object layer - config=%v errMsg=%v", *ncfg, err)
+	}
+
+	// All servers (including local) are told to update in-memory config
+	S3PeersUpdateBucketNotification(bucket, ncfg)
+
+	return nil
+}
+
+// writeNotification marshals notification message before writing to client.
+func writeNotification(w http.ResponseWriter, notification map[string][]NotificationEvent) error {
+	// Invalid response writer.
+	if w == nil {
+		return errInvalidArgument
+	}
+	// Invalid notification input.
+	if notification == nil {
+		return errInvalidArgument
+	}
+	// Marshal notification data into JSON and write to client.
+	notificationBytes, err := json.Marshal(&notification)
+	if err != nil {
+		return err
+	}
+	// Add additional CRLF characters for client to
+	// differentiate the individual events properly.
+	_, err = w.Write(append(notificationBytes, crlf...))
+	// Make sure we have flushed, this would set Transfer-Encoding: chunked.
+	w.(http.Flusher).Flush()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// CRLF character used for chunked transfer in accordance with HTTP standards.
+var crlf = []byte("\r\n")
+
+// sendBucketNotification - writes notification back to client on the response writer
+// for each notification input, otherwise writes whitespace characters periodically
+// to keep the connection active. Each notification messages are terminated by CRLF
+// character. Upon any error received on response writer the for loop exits.
+func sendBucketNotification(w http.ResponseWriter, arnListenerCh <-chan []NotificationEvent) {
+	var dummyEvents = map[string][]NotificationEvent{"Records": nil}
+	// Continuously write to client either timely empty structures
+	// every 5 seconds, or return back the notifications.
+	for {
+		select {
+		case events := <-arnListenerCh:
+			if err := writeNotification(w, map[string][]NotificationEvent{"Records": events}); err != nil {
+				errorIf(err, "Unable to write notification to client.")
+				return
+			}
+		case <-time.After(globalSNSConnAlive): // Wait for global conn active seconds.
+			if err := writeNotification(w, dummyEvents); err != nil {
+				// FIXME - do not log for all errors.
+				errorIf(err, "Unable to write notification to client.")
+				return
+			}
+		}
+	}
+}
+
+// ListenBucketNotificationHandler - list bucket notifications.
+func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
+	// Validate if bucket exists.
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Parse listen bucket notification resources.
+	prefixes, suffixes, events := getListenBucketNotificationResources(r.URL.Query())
+
+	if err := validateFilterValues(prefixes); err != ErrNone {
+		writeErrorResponse(w, r, err, r.URL.Path)
+		return
+	}
+
+	if err := validateFilterValues(suffixes); err != ErrNone {
+		writeErrorResponse(w, r, err, r.URL.Path)
+		return
+	}
+
+	// Validate all the resource events.
+	for _, event := range events {
+		if errCode := checkEvent(event); errCode != ErrNone {
+			writeErrorResponse(w, r, errCode, r.URL.Path)
+			return
+		}
+	}
+
+	_, err := objAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err, "Unable to get bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	accountID := fmt.Sprintf("%d", time.Now().UTC().UnixNano())
+	accountARN := fmt.Sprintf(
+		"%s:%s:%s:%s-%s",
+		minioTopic,
+		serverConfig.GetRegion(),
+		accountID,
+		snsTypeMinio,
+		globalMinioAddr,
+	)
+	var filterRules []filterRule
+
+	for _, prefix := range prefixes {
+		filterRules = append(filterRules, filterRule{
+			Name:  "prefix",
+			Value: prefix,
+		})
+	}
+
+	for _, suffix := range suffixes {
+		filterRules = append(filterRules, filterRule{
+			Name:  "suffix",
+			Value: suffix,
+		})
+	}
+
+	// Make topic configuration corresponding to this
+	// ListenBucketNotification request.
+	topicCfg := &topicConfig{
+		TopicARN: accountARN,
+		ServiceConfig: ServiceConfig{
+			Events: events,
+			Filter: struct {
+				Key keyFilter `xml:"S3Key,omitempty" json:"S3Key,omitempty"`
+			}{
+				Key: keyFilter{
+					FilterRules: filterRules,
+				},
+			},
+			ID: "sns-" + accountID,
+		},
+	}
+
+	// Setup a listening channel that will receive notifications
+	// from the RPC handler.
+	nEventCh := make(chan []NotificationEvent)
+	defer close(nEventCh)
+	// Add channel for listener events
+	if err = globalEventNotifier.AddListenerChan(accountARN, nEventCh); err != nil {
+		errorIf(err, "Error adding a listener!")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// Remove listener channel after the writer has closed or the
+	// client disconnected.
+	defer globalEventNotifier.RemoveListenerChan(accountARN)
+
+	// Update topic config to bucket config and persist - as soon
+	// as this call compelets, events may start appearing in
+	// nEventCh
+	lc := listenerConfig{
+		TopicConfig:  *topicCfg,
+		TargetServer: globalMinioAddr,
+	}
+
+	err = AddBucketListenerConfig(bucket, &lc, objAPI)
+	if err != nil {
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	defer RemoveBucketListenerConfig(bucket, &lc, objAPI)
+
+	// Add all common headers.
+	setCommonHeaders(w)
+
+	// Start sending bucket notifications.
+	sendBucketNotification(w, nEventCh)
+}
+
+// AddBucketListenerConfig - Updates on disk state of listeners, and
+// updates all peers with the change in listener config.
+func AddBucketListenerConfig(bucket string, lcfg *listenerConfig, objAPI ObjectLayer) error {
+	if lcfg == nil {
+		return errInvalidArgument
+	}
+	listenerCfgs := globalEventNotifier.GetBucketListenerConfig(bucket)
+
+	// add new lid to listeners and persist to object layer.
+	listenerCfgs = append(listenerCfgs, *lcfg)
+
+	// Acquire a write lock on bucket before modifying its
+	// configuration.
+	bucketLock := globalNSMutex.NewNSLock(bucket, "")
+	bucketLock.Lock()
+	// Release lock after notifying peers
+	defer bucketLock.Unlock()
+
+	// update persistent config if dist XL
+	if globalIsDistXL {
+		err := persistListenerConfig(bucket, listenerCfgs, objAPI)
+		if err != nil {
+			errorIf(err, "Error persisting listener config when adding a listener.")
+			return err
+		}
+	}
+
+	// persistence success - now update in-memory globals on all
+	// peers (including local)
+	S3PeersUpdateBucketListener(bucket, listenerCfgs)
+	return nil
+}
+
+// RemoveBucketListenerConfig - removes a given bucket notification config
+func RemoveBucketListenerConfig(bucket string, lcfg *listenerConfig, objAPI ObjectLayer) {
+	listenerCfgs := globalEventNotifier.GetBucketListenerConfig(bucket)
+
+	// remove listener with matching ARN - if not found ignore and
+	// exit.
+	var updatedLcfgs []listenerConfig
+	found := false
+	for k, configuredLcfg := range listenerCfgs {
+		if configuredLcfg.TopicConfig.TopicARN == lcfg.TopicConfig.TopicARN {
+			updatedLcfgs = append(listenerCfgs[:k],
+				listenerCfgs[k+1:]...)
+			found = true
+			break
+		}
+	}
+	if !found {
+		return
+	}
+
+	// Acquire a write lock on bucket before modifying its
+	// configuration.
+	bucketLock := globalNSMutex.NewNSLock(bucket, "")
+	bucketLock.Lock()
+	// Release lock after notifying peers
+	defer bucketLock.Unlock()
+
+	// update persistent config if dist XL
+	if globalIsDistXL {
+		err := persistListenerConfig(bucket, updatedLcfgs, objAPI)
+		if err != nil {
+			errorIf(err, "Error persisting listener config when removing a listener.")
+			return
+		}
+	}
+
+	// persistence success - now update in-memory globals on all
+	// peers (including local)
+	S3PeersUpdateBucketListener(bucket, updatedLcfgs)
+}

cmd/bucket-notification-handlers_test.go

@@ -0,0 +1,323 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+)
+
+// Implement a dummy flush writer.
+type flushWriter struct {
+	io.Writer
+}
+
+// Flush writer is a dummy writer compatible with http.Flusher and http.ResponseWriter.
+func (f *flushWriter) Flush()                            {}
+func (f *flushWriter) Write(b []byte) (n int, err error) { return f.Writer.Write(b) }
+func (f *flushWriter) Header() http.Header               { return http.Header{} }
+func (f *flushWriter) WriteHeader(code int)              {}
+
+func newFlushWriter(writer io.Writer) http.ResponseWriter {
+	return &flushWriter{writer}
+}
+
+// Tests write notification code.
+func TestWriteNotification(t *testing.T) {
+	// Initialize a new test config.
+	root, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Unable to initialize test config %s", err)
+	}
+	defer removeAll(root)
+
+	var buffer bytes.Buffer
+	// Collection of test cases for each event writer.
+	testCases := []struct {
+		writer http.ResponseWriter
+		event  map[string][]NotificationEvent
+		err    error
+	}{
+		// Invalid input argument with writer `nil` - Test - 1
+		{
+			writer: nil,
+			event:  nil,
+			err:    errInvalidArgument,
+		},
+		// Invalid input argument with event `nil` - Test - 2
+		{
+			writer: newFlushWriter(ioutil.Discard),
+			event:  nil,
+			err:    errInvalidArgument,
+		},
+		// Unmarshal and write, validate last 5 bytes. - Test - 3
+		{
+			writer: newFlushWriter(&buffer),
+			event: map[string][]NotificationEvent{
+				"Records": {newNotificationEvent(eventData{
+					Type:   ObjectCreatedPut,
+					Bucket: "testbucket",
+					ObjInfo: ObjectInfo{
+						Name: "key",
+					},
+					ReqParams: map[string]string{
+						"ip": "10.1.10.1",
+					}}),
+				},
+			},
+			err: nil,
+		},
+	}
+	// Validates all the testcases for writing notification.
+	for _, testCase := range testCases {
+		err := writeNotification(testCase.writer, testCase.event)
+		if err != testCase.err {
+			t.Errorf("Unable to write notification %s", err)
+		}
+		// Validates if the ending string has 'crlf'
+		if err == nil && !bytes.HasSuffix(buffer.Bytes(), crlf) {
+			buf := buffer.Bytes()[buffer.Len()-5 : 0]
+			t.Errorf("Invalid suffix found from the writer last 5 bytes %s, expected `\r\n`", string(buf))
+		}
+		// Not printing 'buf' on purpose, validates look for string '10.1.10.1'.
+		if err == nil && !bytes.Contains(buffer.Bytes(), []byte("10.1.10.1")) {
+			// Enable when debugging)
+			// fmt.Println(string(buffer.Bytes()))
+			t.Errorf("Requested content couldn't be found, expected `10.1.10.1`")
+		}
+	}
+}
+
+func TestSendBucketNotification(t *testing.T) {
+	// Initialize a new test config.
+	root, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Unable to initialize test config %s", err)
+	}
+	defer removeAll(root)
+
+	eventCh := make(chan []NotificationEvent)
+
+	// Create a Pipe with FlushWriter on the write-side and bufio.Scanner
+	// on the reader-side to receive notification over the listen channel in a
+	// synchronized manner.
+	pr, pw := io.Pipe()
+	fw := newFlushWriter(pw)
+	scanner := bufio.NewScanner(pr)
+	// Start a go-routine to wait for notification events.
+	go func(listenerCh <-chan []NotificationEvent) {
+		sendBucketNotification(fw, listenerCh)
+	}(eventCh)
+
+	// Construct notification events to be passed on the events channel.
+	var events []NotificationEvent
+	evTypes := []EventName{
+		ObjectCreatedPut,
+		ObjectCreatedPost,
+		ObjectCreatedCopy,
+		ObjectCreatedCompleteMultipartUpload,
+	}
+	for _, evType := range evTypes {
+		events = append(events, newNotificationEvent(eventData{
+			Type: evType,
+		}))
+	}
+	// Send notification events to the channel on which sendBucketNotification
+	// is waiting on.
+	eventCh <- events
+
+	// Read from the pipe connected to the ResponseWriter.
+	scanner.Scan()
+	notificationBytes := scanner.Bytes()
+
+	// Close the read-end and send an empty notification event on the channel
+	// to signal sendBucketNotification to terminate.
+	pr.Close()
+	eventCh <- []NotificationEvent{}
+	close(eventCh)
+
+	// Checking if the notification are the same as those sent over the channel.
+	var notifications map[string][]NotificationEvent
+	err = json.Unmarshal(notificationBytes, &notifications)
+	if err != nil {
+		t.Fatal("Failed to Unmarshal notification")
+	}
+	records := notifications["Records"]
+	for i, rec := range records {
+		if rec.EventName == evTypes[i].String() {
+			continue
+		}
+		t.Errorf("Failed to receive %d event %s", i, evTypes[i].String())
+	}
+}
+
+func TestGetBucketNotificationHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testGetBucketNotificationHandler, []string{
+		"GetBucketNotification",
+	})
+}
+
+func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	// declare sample configs
+	filterRules := []filterRule{
+		{
+			Name:  "prefix",
+			Value: "minio",
+		},
+		{
+			Name:  "suffix",
+			Value: "*.jpg",
+		},
+	}
+	sampleSvcCfg := ServiceConfig{
+		[]string{"s3:ObjectRemoved:*", "s3:ObjectCreated:*"},
+		filterStruct{
+			keyFilter{filterRules},
+		},
+		"1",
+	}
+	sampleNotifCfg := notificationConfig{
+		QueueConfigs: []queueConfig{
+			{
+				ServiceConfig: sampleSvcCfg,
+				QueueARN:      "testqARN",
+			},
+		},
+	}
+	rec := httptest.NewRecorder()
+	req, err := newTestSignedRequestV4("GET", getGetBucketNotificationURL("", bucketName),
+		0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+	if err != nil {
+		t.Fatalf("%s: Failed to create HTTP testRequest for ListenBucketNotification: <ERROR> %v", instanceType, err)
+	}
+	apiRouter.ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("Unexpected http response %d", rec.Code)
+	}
+	if err = persistNotificationConfig(bucketName, &sampleNotifCfg, obj); err != nil {
+		t.Fatalf("Unable to save notification config %s", err)
+	}
+	rec = httptest.NewRecorder()
+	req, err = newTestSignedRequestV4("GET", getGetBucketNotificationURL("", bucketName),
+		0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+	if err != nil {
+		t.Fatalf("%s: Failed to create HTTP testRequest for ListenBucketNotification: <ERROR> %v", instanceType, err)
+	}
+	apiRouter.ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("Unexpected http response %d", rec.Code)
+	}
+	notificationBytes, err := ioutil.ReadAll(rec.Body)
+	if err != nil {
+		t.Fatalf("Unexpected error %s", err)
+	}
+	nConfig := notificationConfig{}
+	if err = xml.Unmarshal(notificationBytes, &nConfig); err != nil {
+		t.Fatalf("Unexpected XML received %s", err)
+	}
+	if sampleNotifCfg.QueueConfigs[0].QueueARN != nConfig.QueueConfigs[0].QueueARN {
+		t.Fatalf("Uexpected notification configs expected %#v, got %#v", sampleNotifCfg, nConfig)
+	}
+	if !reflect.DeepEqual(sampleNotifCfg.QueueConfigs[0].Events, nConfig.QueueConfigs[0].Events) {
+		t.Fatalf("Uexpected notification configs expected %#v, got %#v", sampleNotifCfg, nConfig)
+	}
+}
+
+func TestListenBucketNotificationNilHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testListenBucketNotificationNilHandler, []string{
+		"ListenBucketNotification",
+		"PutObject",
+	})
+}
+
+func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	// get random bucket name.
+	randBucket := getRandomBucketName()
+
+	// Nil Object layer
+	nilAPIRouter := initTestAPIEndPoints(nil, []string{
+		"ListenBucketNotification",
+	})
+	testRec := httptest.NewRecorder()
+	testReq, tErr := newTestSignedRequestV4("GET",
+		getListenBucketNotificationURL("", randBucket, []string{},
+			[]string{"*.jpg"}, []string{
+				"s3:ObjectCreated:*",
+				"s3:ObjectRemoved:*",
+			}), 0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+	if tErr != nil {
+		t.Fatalf("%s: Failed to create HTTP testRequest for ListenBucketNotification: <ERROR> %v", instanceType, tErr)
+	}
+	nilAPIRouter.ServeHTTP(testRec, testReq)
+	if testRec.Code != http.StatusServiceUnavailable {
+		t.Fatalf("Test 1: %s: expected HTTP code %d, but received %d: <ERROR> %v",
+			instanceType, http.StatusServiceUnavailable, testRec.Code, tErr)
+	}
+}
+
+func testRemoveNotificationConfig(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	invalidBucket := "Invalid\\Bucket"
+	// get random bucket name.
+	randBucket := bucketName
+
+	sampleNotificationBytes := []byte("<NotificationConfiguration><TopicConfiguration>" +
+		"<Event>s3:ObjectCreated:*</Event><Event>s3:ObjectRemoved:*</Event><Filter>" +
+		"<S3Key></S3Key></Filter><Id></Id><Topic>arn:minio:sns:us-east-1:1474332374:listen</Topic>" +
+		"</TopicConfiguration></NotificationConfiguration>")
+
+	// Set sample bucket notification on randBucket.
+	testRec := httptest.NewRecorder()
+	testReq, tErr := newTestSignedRequestV4("PUT", getPutBucketNotificationURL("", randBucket),
+		int64(len(sampleNotificationBytes)), bytes.NewReader(sampleNotificationBytes),
+		credentials.AccessKeyID, credentials.SecretAccessKey)
+	if tErr != nil {
+		t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v", instanceType, tErr)
+	}
+	apiRouter.ServeHTTP(testRec, testReq)
+
+	testCases := []struct {
+		bucketName  string
+		expectedErr error
+	}{
+		{invalidBucket, BucketNameInvalid{Bucket: invalidBucket}},
+		{randBucket, nil},
+	}
+	for i, test := range testCases {
+		tErr := removeNotificationConfig(test.bucketName, obj)
+		if tErr != test.expectedErr {
+			t.Errorf("Test %d: %s expected error %v, but received %v", i+1, instanceType, test.expectedErr, tErr)
+		}
+	}
+}
+
+func TestRemoveNotificationConfig(t *testing.T) {
+	ExecObjectLayerAPITest(t, testRemoveNotificationConfig, []string{
+		"PutBucketNotification",
+		"ListenBucketNotification",
+	})
+}

cmd/bucket-notification-utils.go

@@ -0,0 +1,259 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "strings"
+
+// List of valid event types.
+var suppportedEventTypes = map[string]struct{}{
+	// Object created event types.
+	"s3:ObjectCreated:*":                       {},
+	"s3:ObjectCreated:Put":                     {},
+	"s3:ObjectCreated:Post":                    {},
+	"s3:ObjectCreated:Copy":                    {},
+	"s3:ObjectCreated:CompleteMultipartUpload": {},
+	// Object removed event types.
+	"s3:ObjectRemoved:*":      {},
+	"s3:ObjectRemoved:Delete": {},
+}
+
+// checkEvent - checks if an event is supported.
+func checkEvent(event string) APIErrorCode {
+	_, ok := suppportedEventTypes[event]
+	if !ok {
+		return ErrEventNotification
+	}
+	return ErrNone
+}
+
+// checkEvents - checks given list of events if all of them are valid.
+// given if one of them is invalid, this function returns an error.
+func checkEvents(events []string) APIErrorCode {
+	for _, event := range events {
+		if s3Error := checkEvent(event); s3Error != ErrNone {
+			return s3Error
+		}
+	}
+	return ErrNone
+}
+
+// Valid if filterName is 'prefix'.
+func isValidFilterNamePrefix(filterName string) bool {
+	return "prefix" == filterName
+}
+
+// Valid if filterName is 'suffix'.
+func isValidFilterNameSuffix(filterName string) bool {
+	return "suffix" == filterName
+}
+
+// Is this a valid filterName? - returns true if valid.
+func isValidFilterName(filterName string) bool {
+	return isValidFilterNamePrefix(filterName) || isValidFilterNameSuffix(filterName)
+}
+
+// checkFilterRules - checks given list of filter rules if all of them are valid.
+func checkFilterRules(filterRules []filterRule) APIErrorCode {
+	ruleSetMap := make(map[string]string)
+	// Validate all filter rules.
+	for _, filterRule := range filterRules {
+		// Unknown filter rule name found, returns an appropriate error.
+		if !isValidFilterName(filterRule.Name) {
+			return ErrFilterNameInvalid
+		}
+
+		// Filter names should not be set twice per notification service
+		// configuration, if found return an appropriate error.
+		if _, ok := ruleSetMap[filterRule.Name]; ok {
+			if isValidFilterNamePrefix(filterRule.Name) {
+				return ErrFilterNamePrefix
+			} else if isValidFilterNameSuffix(filterRule.Name) {
+				return ErrFilterNameSuffix
+			} else {
+				return ErrFilterNameInvalid
+			}
+		}
+
+		if !IsValidObjectPrefix(filterRule.Value) {
+			return ErrFilterValueInvalid
+		}
+
+		// Set the new rule name to keep track of duplicates.
+		ruleSetMap[filterRule.Name] = filterRule.Value
+	}
+	// Success all prefixes validated.
+	return ErrNone
+}
+
+// Checks validity of input ARN for a given arnType.
+func checkARN(arn, arnType string) APIErrorCode {
+	if !strings.HasPrefix(arn, arnType) {
+		return ErrARNNotification
+	}
+	if !strings.HasPrefix(arn, arnType+serverConfig.GetRegion()+":") {
+		return ErrRegionNotification
+	}
+	account := strings.SplitN(strings.TrimPrefix(arn, arnType+serverConfig.GetRegion()+":"), ":", 2)
+	switch len(account) {
+	case 1:
+		// This means ARN is malformed, account should have min of 2elements.
+		return ErrARNNotification
+	case 2:
+		// Account topic id or topic name cannot be empty.
+		if account[0] == "" || account[1] == "" {
+			return ErrARNNotification
+		}
+	}
+	return ErrNone
+}
+
+// checkQueueARN - check if the queue arn is valid.
+func checkQueueARN(queueARN string) APIErrorCode {
+	return checkARN(queueARN, minioSqs)
+}
+
+// Validates account id for input queue ARN.
+func isValidQueueID(queueARN string) bool {
+	// Unmarshals QueueARN into structured object.
+	sqsARN := unmarshalSqsARN(queueARN)
+	// Is Queue identifier valid?.
+	if isAMQPQueue(sqsARN) { // AMQP eueue.
+		amqpN := serverConfig.GetAMQPNotifyByID(sqsARN.AccountID)
+		return amqpN.Enable && amqpN.URL != ""
+	} else if isNATSQueue(sqsARN) {
+		natsN := serverConfig.GetNATSNotifyByID(sqsARN.AccountID)
+		return natsN.Enable && natsN.Address != ""
+	} else if isElasticQueue(sqsARN) { // Elastic queue.
+		elasticN := serverConfig.GetElasticSearchNotifyByID(sqsARN.AccountID)
+		return elasticN.Enable && elasticN.URL != ""
+	} else if isRedisQueue(sqsARN) { // Redis queue.
+		redisN := serverConfig.GetRedisNotifyByID(sqsARN.AccountID)
+		return redisN.Enable && redisN.Addr != ""
+	} else if isPostgreSQLQueue(sqsARN) {
+		pgN := serverConfig.GetPostgreSQLNotifyByID(sqsARN.AccountID)
+		// Postgres can work with only default conn. info.
+		return pgN.Enable
+	}
+	return false
+}
+
+// Check - validates queue configuration and returns error if any.
+func checkQueueConfig(qConfig queueConfig) APIErrorCode {
+	// Check queue arn is valid.
+	if s3Error := checkQueueARN(qConfig.QueueARN); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Validate if the account ID is correct.
+	if !isValidQueueID(qConfig.QueueARN) {
+		return ErrARNNotification
+	}
+
+	// Check if valid events are set in queue config.
+	if s3Error := checkEvents(qConfig.Events); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Check if valid filters are set in queue config.
+	if s3Error := checkFilterRules(qConfig.Filter.Key.FilterRules); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Success.
+	return ErrNone
+}
+
+// Validates all incoming queue configs, checkQueueConfig validates if the
+// input fields for each queues is not malformed and has valid configuration
+// information.  If validation fails bucket notifications are not enabled.
+func validateQueueConfigs(queueConfigs []queueConfig) APIErrorCode {
+	for _, qConfig := range queueConfigs {
+		if s3Error := checkQueueConfig(qConfig); s3Error != ErrNone {
+			return s3Error
+		}
+	}
+	// Success.
+	return ErrNone
+}
+
+// Check all the queue configs for any duplicates.
+func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
+	var queueConfigARNS []string
+
+	// Navigate through each configs and count the entries.
+	for _, config := range configs {
+		queueConfigARNS = append(queueConfigARNS, config.QueueARN)
+	}
+
+	// Check if there are any duplicate counts.
+	if err := checkDuplicateStrings(queueConfigARNS); err != nil {
+		errorIf(err, "Invalid queue configs found.")
+		return ErrOverlappingConfigs
+	}
+
+	// Success.
+	return ErrNone
+}
+
+// Validates all the bucket notification configuration for their validity,
+// if one of the config is malformed or has invalid data it is rejected.
+// Configuration is never applied partially.
+func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
+	// Validate all queue configs.
+	if s3Error := validateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Check for duplicate queue configs.
+	if len(nConfig.QueueConfigs) > 1 {
+		if s3Error := checkDuplicateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
+			return s3Error
+		}
+	}
+
+	// Add validation for other configurations.
+	return ErrNone
+}
+
+// Unmarshals input value of AWS ARN format into minioSqs object.
+// Returned value represents minio sqs types, currently supported are
+// - amqp
+// - nats
+// - elasticsearch
+// - redis
+// - postgresql
+func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
+	mSqs = arnSQS{}
+	if !strings.HasPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":") {
+		return mSqs
+	}
+	sqsType := strings.TrimPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":")
+	switch {
+	case strings.HasSuffix(sqsType, queueTypeAMQP):
+		mSqs.Type = queueTypeAMQP
+	case strings.HasSuffix(sqsType, queueTypeNATS):
+		mSqs.Type = queueTypeNATS
+	case strings.HasSuffix(sqsType, queueTypeElastic):
+		mSqs.Type = queueTypeElastic
+	case strings.HasSuffix(sqsType, queueTypeRedis):
+		mSqs.Type = queueTypeRedis
+	case strings.HasSuffix(sqsType, queueTypePostgreSQL):
+		mSqs.Type = queueTypePostgreSQL
+	} // Add more queues here.
+	mSqs.AccountID = strings.TrimSuffix(sqsType, ":"+mSqs.Type)
+	return mSqs
+}

cmd/bucket-notification-utils_test.go

@@ -0,0 +1,353 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"strings"
+	"testing"
+)
+
+// Test validates for duplicate configs.
+func TestCheckDuplicateConfigs(t *testing.T) {
+	testCases := []struct {
+		qConfigs        []queueConfig
+		expectedErrCode APIErrorCode
+	}{
+		// Error for duplicate queue configs.
+		{
+			qConfigs: []queueConfig{
+				{
+					QueueARN: "arn:minio:sqs:us-east-1:1:redis",
+				},
+				{
+					QueueARN: "arn:minio:sqs:us-east-1:1:redis",
+				},
+			},
+			expectedErrCode: ErrOverlappingConfigs,
+		},
+		// Valid queue configs.
+		{
+			qConfigs: []queueConfig{
+				{
+					QueueARN: "arn:minio:sqs:us-east-1:1:redis",
+				},
+			},
+			expectedErrCode: ErrNone,
+		},
+	}
+
+	// ... validate for duplicate queue configs.
+	for i, testCase := range testCases {
+		errCode := checkDuplicateQueueConfigs(testCase.qConfigs)
+		if errCode != testCase.expectedErrCode {
+			t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.expectedErrCode, errCode)
+		}
+	}
+}
+
+// Tests for validating filter rules.
+func TestCheckFilterRules(t *testing.T) {
+	testCases := []struct {
+		rules           []filterRule
+		expectedErrCode APIErrorCode
+	}{
+		// Valid prefix and suffix values.
+		{
+			rules: []filterRule{
+				{
+					Name:  "prefix",
+					Value: "test/test1",
+				},
+				{
+					Name:  "suffix",
+					Value: ".jpg",
+				},
+			},
+			expectedErrCode: ErrNone,
+		},
+		// Invalid filter name.
+		{
+			rules: []filterRule{
+				{
+					Name:  "unknown",
+					Value: "test/test1",
+				},
+			},
+			expectedErrCode: ErrFilterNameInvalid,
+		},
+		// Cannot have duplicate prefixes.
+		{
+			rules: []filterRule{
+				{
+					Name:  "prefix",
+					Value: "test/test1",
+				},
+				{
+					Name:  "prefix",
+					Value: "test/test1",
+				},
+			},
+			expectedErrCode: ErrFilterNamePrefix,
+		},
+		// Cannot have duplicate suffixes.
+		{
+			rules: []filterRule{
+				{
+					Name:  "suffix",
+					Value: ".jpg",
+				},
+				{
+					Name:  "suffix",
+					Value: ".txt",
+				},
+			},
+			expectedErrCode: ErrFilterNameSuffix,
+		},
+		// Filter value cannot be bigger than > 1024.
+		{
+			rules: []filterRule{
+				{
+					Name:  "prefix",
+					Value: strings.Repeat("a", 1025),
+				},
+			},
+			expectedErrCode: ErrFilterValueInvalid,
+		},
+	}
+
+	for i, testCase := range testCases {
+		errCode := checkFilterRules(testCase.rules)
+		if errCode != testCase.expectedErrCode {
+			t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.expectedErrCode, errCode)
+		}
+	}
+}
+
+// Tests filter name validation.
+func TestIsValidFilterName(t *testing.T) {
+	testCases := []struct {
+		filterName string
+		status     bool
+	}{
+		// Validate if 'prefix' is correct.
+		{
+			filterName: "prefix",
+			status:     true,
+		},
+		// Validate if 'suffix' is correct.
+		{
+			filterName: "suffix",
+			status:     true,
+		},
+		// Invalid filter name empty string should return false.
+		{
+			filterName: "",
+			status:     false,
+		},
+		// Invalid filter name random character should return false.
+		{
+			filterName: "unknown",
+			status:     false,
+		},
+	}
+
+	for i, testCase := range testCases {
+		status := isValidFilterName(testCase.filterName)
+		if testCase.status != status {
+			t.Errorf("Test %d: Expected \"%t\", got \"%t\"", i+1, testCase.status, status)
+		}
+	}
+}
+
+// Tests list of valid and invalid events.
+func TestValidEvents(t *testing.T) {
+	testCases := []struct {
+		events  []string
+		errCode APIErrorCode
+	}{
+		// Return error for unknown event element.
+		{
+			events: []string{
+				"s3:UnknownAPI",
+			},
+			errCode: ErrEventNotification,
+		},
+		// Return success for supported event.
+		{
+			events: []string{
+				"s3:ObjectCreated:Put",
+			},
+			errCode: ErrNone,
+		},
+		// Return success for supported events.
+		{
+			events: []string{
+				"s3:ObjectCreated:*",
+				"s3:ObjectRemoved:*",
+			},
+			errCode: ErrNone,
+		},
+		// Return error for empty event list.
+		{
+			events:  []string{""},
+			errCode: ErrEventNotification,
+		},
+	}
+
+	for i, testCase := range testCases {
+		errCode := checkEvents(testCase.events)
+		if testCase.errCode != errCode {
+			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
+		}
+	}
+}
+
+// Tests queue arn validation.
+func TestQueueARN(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases := []struct {
+		queueARN string
+		errCode  APIErrorCode
+	}{
+		// Valid redis queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:redis",
+			errCode:  ErrNone,
+		},
+		// Valid elasticsearch queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:elasticsearch",
+			errCode:  ErrNone,
+		},
+		// Valid amqp queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:amqp",
+			errCode:  ErrNone,
+		},
+		// Invalid empty queue arn.
+		{
+			queueARN: "",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid notification service type.
+		{
+			queueARN: "arn:minio:sns:us-east-1:1:listen",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid region 'us-west-1' in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:redis",
+			errCode:  ErrRegionNotification,
+		},
+		// Invalid queue name empty in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid queue id empty in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1::redis",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid queue id and queue name empty in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1::",
+			errCode:  ErrARNNotification,
+		},
+		// Missing queue id and separator missing at the end in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:amqp",
+			errCode:  ErrARNNotification,
+		},
+		// Missing queue id and empty string at the end in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:",
+			errCode:  ErrARNNotification,
+		},
+	}
+
+	// Validate all tests for queue arn.
+	for i, testCase := range testCases {
+		errCode := checkQueueARN(testCase.queueARN)
+		if testCase.errCode != errCode {
+			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
+		}
+	}
+}
+
+// Test unmarshal queue arn.
+func TestUnmarshalSQSARN(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases := []struct {
+		queueARN string
+		Type     string
+	}{
+		// Valid redis queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:redis",
+			Type:     "redis",
+		},
+		// Valid elasticsearch queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:elasticsearch",
+			Type:     "elasticsearch",
+		},
+		// Valid amqp queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:amqp",
+			Type:     "amqp",
+		},
+		// Invalid empty queue arn.
+		{
+			queueARN: "",
+			Type:     "",
+		},
+		// Invalid region 'us-west-1' in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:redis",
+			Type:     "",
+		},
+		// Partial queue arn.
+		{
+			queueARN: "arn:minio:sqs:",
+			Type:     "",
+		},
+		// Invalid queue service value.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:*",
+			Type:     "",
+		},
+	}
+
+	for i, testCase := range testCases {
+		mSqs := unmarshalSqsARN(testCase.queueARN)
+		if testCase.Type != mSqs.Type {
+			t.Errorf("Test %d: Expected \"%s\", got \"%s\"", i+1, testCase.Type, mSqs.Type)
+		}
+	}
+
+}

cmd/bucket-policy-handlers.go

@@ -0,0 +1,264 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	humanize "github.com/dustin/go-humanize"
+	mux "github.com/gorilla/mux"
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/wildcard"
+)
+
+// maximum supported access policy size.
+const maxAccessPolicySize = 20 * humanize.KiByte
+
+// Verify if a given action is valid for the url path based on the
+// existing bucket access policy.
+func bucketPolicyEvalStatements(action string, resource string, conditions map[string]set.StringSet, statements []policyStatement) bool {
+	for _, statement := range statements {
+		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
+			if statement.Effect == "Allow" {
+				return true
+			}
+			// Do not uncomment kept here for readability.
+			// else statement.Effect == "Deny"
+			return false
+		}
+	}
+	// None match so deny.
+	return false
+}
+
+// Verify if action, resource and conditions match input policy statement.
+func bucketPolicyMatchStatement(action string, resource string, conditions map[string]set.StringSet, statement policyStatement) bool {
+	// Verify if action, resource and condition match in given statement.
+	return (bucketPolicyActionMatch(action, statement) &&
+		bucketPolicyResourceMatch(resource, statement) &&
+		bucketPolicyConditionMatch(conditions, statement))
+}
+
+// Verify if given action matches with policy statement.
+func bucketPolicyActionMatch(action string, statement policyStatement) bool {
+	return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
+}
+
+// Match function matches wild cards in 'pattern' for resource.
+func resourceMatch(pattern, resource string) bool {
+	return wildcard.Match(pattern, resource)
+}
+
+// Match function matches wild cards in 'pattern' for action.
+func actionMatch(pattern, action string) bool {
+	return wildcard.MatchSimple(pattern, action)
+}
+
+// Verify if given resource matches with policy statement.
+func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
+	// the resource rule for object could contain "*" wild card.
+	// the requested object can be given access based on the already set bucket policy if
+	// the match is successful.
+	// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html.
+	return !statement.Resources.FuncMatch(resourceMatch, resource).IsEmpty()
+}
+
+// Verify if given condition matches with policy statement.
+func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement policyStatement) bool {
+	// Supports following conditions.
+	// - StringEquals
+	// - StringNotEquals
+	//
+	// Supported applicable condition keys for each conditions.
+	// - s3:prefix
+	// - s3:max-keys
+	var conditionMatches = true
+	for condition, conditionKeyVal := range statement.Conditions {
+		if condition == "StringEquals" {
+			if !conditionKeyVal["s3:prefix"].Equals(conditions["prefix"]) {
+				conditionMatches = false
+				break
+			}
+			if !conditionKeyVal["s3:max-keys"].Equals(conditions["max-keys"]) {
+				conditionMatches = false
+				break
+			}
+		} else if condition == "StringNotEquals" {
+			if !conditionKeyVal["s3:prefix"].Equals(conditions["prefix"]) {
+				conditionMatches = false
+				break
+			}
+			if !conditionKeyVal["s3:max-keys"].Equals(conditions["max-keys"]) {
+				conditionMatches = false
+				break
+			}
+		}
+	}
+	return conditionMatches
+}
+
+// PutBucketPolicyHandler - PUT Bucket policy
+// -----------------
+// This implementation of the PUT operation uses the policy
+// subresource to add to or replace a policy on a bucket
+func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Before proceeding validate if bucket exists.
+	_, err := objAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err, "Unable to find bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// If Content-Length is unknown or zero, deny the
+	// request. PutBucketPolicy always needs a Content-Length if
+	// incoming request is not chunked.
+	if !contains(r.TransferEncoding, "chunked") {
+		if r.ContentLength == -1 || r.ContentLength == 0 {
+			writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+			return
+		}
+		// If Content-Length is greater than maximum allowed policy size.
+		if r.ContentLength > maxAccessPolicySize {
+			writeErrorResponse(w, r, ErrEntityTooLarge, r.URL.Path)
+			return
+		}
+	}
+
+	// Read access policy up to maxAccessPolicySize.
+	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+	// bucket policies are limited to 20KB in size, using a limit reader.
+	policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	if err != nil {
+		errorIf(err, "Unable to read from client.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Parse validate and save bucket policy.
+	if s3Error := parseAndPersistBucketPolicy(bucket, policyBytes, objAPI); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Success.
+	writeSuccessNoContent(w)
+}
+
+// DeleteBucketPolicyHandler - DELETE Bucket policy
+// -----------------
+// This implementation of the DELETE operation uses the policy
+// subresource to add to remove a policy on a bucket.
+func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Before proceeding validate if bucket exists.
+	_, err := objAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err, "Unable to find bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Delete bucket access policy, by passing an empty policy
+	// struct.
+	if err := persistAndNotifyBucketPolicyChange(bucket, policyChange{true, nil}, objAPI); err != nil {
+		switch err.(type) {
+		case BucketPolicyNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+
+	// Success.
+	writeSuccessNoContent(w)
+}
+
+// GetBucketPolicyHandler - GET Bucket policy
+// -----------------
+// This operation uses the policy
+// subresource to return the policy of a specified bucket.
+func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(w, r, ErrServerNotInitialized, r.URL.Path)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Before proceeding validate if bucket exists.
+	_, err := objAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err, "Unable to find bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Read bucket access policy.
+	policy, err := readBucketPolicy(bucket, objAPI)
+	if err != nil {
+		errorIf(err, "Unable to read bucket policy.")
+		switch err.(type) {
+		case BucketPolicyNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+
+	// Write to client.
+	fmt.Fprint(w, policy)
+}

cmd/bucket-policy-handlers_test.go

@@ -0,0 +1,956 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/minio/minio-go/pkg/set"
+)
+
+// Tests validate Bucket policy resource matcher.
+func TestBucketPolicyResourceMatch(t *testing.T) {
+
+	// generates statement with given resource..
+	generateStatement := func(resource string) policyStatement {
+		statement := policyStatement{}
+		statement.Resources = set.CreateStringSet([]string{resource}...)
+		return statement
+	}
+
+	// generates resource prefix.
+	generateResource := func(bucketName, objectName string) string {
+		return AWSResourcePrefix + bucketName + "/" + objectName
+	}
+
+	testCases := []struct {
+		resourceToMatch       string
+		statement             policyStatement
+		expectedResourceMatch bool
+	}{
+		// Test case 1-4.
+		// Policy with resource ending with bucket/* allows access to all objects inside the given bucket.
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		// Test case - 5.
+		// Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt.
+		{generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), false},
+		// Test case - 6.
+		// Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt.
+		{generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true},
+		// Test case - 7.
+		// Policy with resource ending with bucket/oo* allows access to all sub-dirs starting with "oo" inside given bucket.
+		{generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true},
+		// Test case - 8.
+		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false},
+		// Test case - 9.
+		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false},
+		// Test case - 10.
+		// Proves that the name space is flat.
+		{generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix,
+			"minio-bucket"+"/*/India/*/Bihar")), true},
+		// Test case - 11.
+		// Proves that the name space is flat.
+		{generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix,
+			"minio-bucket"+"/*/India/*/Bihar/*")), true},
+	}
+	for i, testCase := range testCases {
+		actualResourceMatch := bucketPolicyResourceMatch(testCase.resourceToMatch, testCase.statement)
+		if testCase.expectedResourceMatch != actualResourceMatch {
+			t.Errorf("Test %d: Expected Resource match to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResourceMatch, actualResourceMatch)
+		}
+	}
+}
+
+// TestBucketPolicyActionMatch - Test validates whether given action on the
+// bucket/object matches the allowed actions in policyStatement.
+// This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
+// The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
+func TestBucketPolicyActionMatch(t *testing.T) {
+	bucketName := getRandomBucketName()
+	objectPrefix := "test-object"
+
+	testCases := []struct {
+		action         string
+		statement      policyStatement
+		expectedResult bool
+	}{
+		// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
+		// fetching of bucket location on an Anonymous/unsigned request.
+
+		//r ead-write bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 1).
+		{"s3:GetBucketLocation", getReadWriteBucketStatement(bucketName, objectPrefix), true},
+		//	write-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 2).
+		{"s3:GetBucketLocation", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
+		//	read-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 3).
+		{"s3:GetBucketLocation", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
+
+		// Any of the Object level access permissions shouldn't allow for GetBucketLocation operation  on an Anonymous/unsigned request (Test cases 4-6).
+		{"s3:GetBucketLocation", getReadWriteObjectStatement(bucketName, objectPrefix), false},
+		{"s3:GetBucketLocation", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+		{"s3:GetBucketLocation", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:ListBucketMultipartUploads is the action necessary to be present in the bucket policy to allow
+		// Listing of multipart uploads in a given bucket for an Anonymous/unsigned request.
+
+		//read-write bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 7).
+		{"s3:ListBucketMultipartUploads", getReadWriteBucketStatement(bucketName, objectPrefix), true},
+		//	write-only bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 8).
+		{"s3:ListBucketMultipartUploads", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
+		// read-only bucket policy is expected to not allow ListBucketMultipartUploads operation on an anonymous request (Test case 9).
+		// the allowed actions in read-only bucket statement are  "s3:GetBucketLocation","s3:ListBucket",
+		// this should not allow for ListBucketMultipartUploads operations.
+		{"s3:ListBucketMultipartUploads", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Any of the object level policy will not allow for s3:ListBucketMultipartUploads (Test cases 10-12).
+		{"s3:ListBucketMultipartUploads", getReadWriteObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucketMultipartUploads", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucketMultipartUploads", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:ListBucket is the action necessary to be present in the bucket policy to allow
+		// listing of all objects inside a given bucket on an Anonymous/unsigned request.
+
+		// Cases for testing ListBucket access for different Bucket level access permissions.
+		// read-only bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 13).
+		{"s3:ListBucket", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
+		// read-write bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 14).
+		{"s3:ListBucket", getReadWriteBucketStatement(bucketName, objectPrefix), true},
+		// write-only  bucket policy is expected to not allow ListBucket operation on an anonymous request (Test case 15).
+		// the allowed actions in write-only  bucket statement are "s3:GetBucketLocation",	"s3:ListBucketMultipartUploads",
+		// this should not allow for ListBucket operations.
+		{"s3:ListBucket", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing ListBucket access for different Object level access permissions (Test cases 16-18).
+		// Any of the Object level access permissions shouldn't allow for ListBucket operation  on an Anonymous/unsigned request.
+		{"s3:ListBucket", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucket", getReadWriteObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucket", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:DeleteObject is the action necessary to be present in the bucket policy to allow
+		// deleting/removal of  objects inside a given bucket for an Anonymous/unsigned request.
+
+		// Cases for testing DeleteObject access for different Bucket level access permissions (Test cases 19-21).
+		// Any of the Bucket level access permissions shouldn't allow for DeleteObject operation  on an Anonymous/unsigned request.
+		{"s3:DeleteObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:DeleteObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:DeleteObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing DeleteObject access for different Object level access permissions (Test cases 22).
+		// read-only bucket policy is expected to not allow Delete Object operation on an anonymous request.
+		{"s3:DeleteObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write bucket policy is expected to allow Delete Bucket operation on an anonymous request (Test cases 23).
+		{"s3:DeleteObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  bucket policy is expected to allow Delete Object operation on an anonymous request (Test cases 24).
+		{"s3:DeleteObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+
+		// s3:AbortMultipartUpload is the action necessary to be present in the bucket policy to allow
+		// cancelling or abortion of an already initiated multipart upload operation for an Anonymous/unsigned request.
+
+		// Cases for testing AbortMultipartUpload access for different Bucket level access permissions (Test cases 25-27).
+		// Any of the Bucket level access permissions shouldn't allow for AbortMultipartUpload operation on an Anonymous/unsigned request.
+		{"s3:AbortMultipartUpload", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:AbortMultipartUpload", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:AbortMultipartUpload", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing AbortMultipartUpload access for different Object level access permissions.
+		// read-only object policy is expected to not allow AbortMultipartUpload operation on an anonymous request (Test case 28).
+		{"s3:AbortMultipartUpload", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 29).
+		{"s3:AbortMultipartUpload", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 30).
+		{"s3:AbortMultipartUpload", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+
+		// s3:PutObject is the action necessary to be present in the bucket policy to allow
+		// uploading of an object for an Anonymous/unsigned request.
+
+		// Cases for testing PutObject access for different Bucket level access permissions (Test cases 31-33).
+		// Any of the Bucket level access permissions shouldn't allow for PutObject operation on an Anonymous/unsigned request.
+		{"s3:PutObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:PutObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:PutObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing PutObject access for different Object level access permissions.
+		// read-only object policy is expected to not allow PutObject operation on an anonymous request (Test case 34).
+		{"s3:PutObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write object policy is expected to allow PutObject operation on an anonymous request (Test case 35).
+		{"s3:PutObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  object policy is expected to allow PutObject operation on an anonymous request (Test case 36).
+		{"s3:PutObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+
+		// s3:GetObject is the action necessary to be present in the bucket policy to allow
+		// downloading of an object for an Anonymous/unsigned request.
+
+		// Cases for testing GetObject access for different Bucket level access permissions (Test cases 37-39).
+		// Any of the Bucket level access permissions shouldn't allow for GetObject operation on an Anonymous/unsigned request.
+		{"s3:GetObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:GetObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:GetObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing GetObject access for different Object level access permissions.
+		// read-only bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 40).
+		{"s3:GetObject", getReadOnlyObjectStatement(bucketName, objectPrefix), true},
+		// read-write bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 41).
+		{"s3:GetObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  bucket policy is expected to not allow  downloading of an Object on an anonymous request (Test case 42).
+		{"s3:GetObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:ListMultipartUploadParts is the action necessary to be present in the bucket policy to allow
+		// Listing of uploaded parts for an Anonymous/unsigned request.
+
+		// Any of the Bucket level access permissions shouldn't allow for ListMultipartUploadParts operation on an Anonymous/unsigned request.
+		// read-only bucket policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test cases 43-45).
+		{"s3:ListMultipartUploadParts", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:ListMultipartUploadParts", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:ListMultipartUploadParts", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// read-only object policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test case 46).
+		{"s3:ListMultipartUploadParts", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 47).
+		{"s3:ListMultipartUploadParts", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 48).
+		{"s3:ListMultipartUploadParts", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+	}
+	for i, testCase := range testCases {
+		actualResult := bucketPolicyActionMatch(testCase.action, testCase.statement)
+		if testCase.expectedResult != actualResult {
+			t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResult, actualResult)
+		}
+	}
+}
+
+// Wrapper for calling Put Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+func TestPutBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testPutBucketPolicyHandler, []string{"PutBucketPolicy"})
+}
+
+// testPutBucketPolicyHandler - Test for Bucket policy end point.
+func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	initBucketPolicies(obj)
+
+	bucketName1 := fmt.Sprintf("%s-1", bucketName)
+	if err := obj.MakeBucket(bucketName1); err != nil {
+		t.Fatal(err)
+	}
+
+	// template for constructing HTTP request body for PUT bucket policy.
+	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
+
+	// test cases with sample input and expected output.
+	testCases := []struct {
+		bucketName string
+		// bucket policy to be set,
+		// set as request body.
+		bucketPolicyReader io.ReadSeeker
+		// length in bytes of the bucket policy being set.
+		policyLen int
+		accessKey string
+		secretKey string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		// Test case - 1.
+		{
+			bucketName:         bucketName,
+			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
+
+			policyLen:          len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNoContent,
+		},
+		// Test case - 2.
+		// Setting the content length to be more than max allowed size.
+		// Expecting StatusBadRequest (400).
+		{
+			bucketName:         bucketName,
+			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
+
+			policyLen:          maxAccessPolicySize + 1,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusBadRequest,
+		},
+		// Test case - 3.
+		// Case with content-length of the HTTP request set to 0.
+		// Expecting the HTTP response status to be StatusLengthRequired (411).
+		{
+			bucketName:         bucketName,
+			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
+
+			policyLen:          0,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusLengthRequired,
+		},
+		// Test case - 4.
+		// setting the readSeeker to `nil`, bucket policy parser will fail.
+		{
+			bucketName:         bucketName,
+			bucketPolicyReader: nil,
+
+			policyLen:          10,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusBadRequest,
+		},
+		// Test case - 5.
+		// setting the keys to be empty.
+		// Expecting statusForbidden.
+		{
+			bucketName:         bucketName,
+			bucketPolicyReader: nil,
+
+			policyLen:          10,
+			accessKey:          "",
+			secretKey:          "",
+			expectedRespStatus: http.StatusForbidden,
+		},
+		// Test case - 6.
+		// setting an invalid bucket policy.
+		// the bucket policy parser will fail.
+		{
+			bucketName:         bucketName,
+			bucketPolicyReader: bytes.NewReader([]byte("dummy-policy")),
+
+			policyLen:          len([]byte("dummy-policy")),
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusBadRequest,
+		},
+		// Test case - 7.
+		// Different bucket name used in the HTTP request and the policy string.
+		// checkBucketPolicyResources should fail.
+		{
+			bucketName:         bucketName1,
+			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
+
+			policyLen:          len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusBadRequest,
+		},
+		// Test case - 8.
+		// non-existent bucket is used.
+		// writing BucketPolicy should fail.
+		// should result is 500 InternalServerError.
+		{
+			bucketName:         "non-existent-bucket",
+			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, "non-existent-bucket", "non-existent-bucket"))),
+
+			policyLen:          len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 9.
+		// invalid bucket name is used.
+		// writing BucketPolicy should fail.
+		// should result is 400 StatusBadRequest.
+		{
+			bucketName:         ".invalid-bucket",
+			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, ".invalid-bucket", ".invalid-bucket"))),
+
+			policyLen:          len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusBadRequest,
+		},
+	}
+
+	// Iterating over the test cases, calling the function under test and asserting the response.
+	for i, testCase := range testCases {
+		// obtain the put bucket policy request body.
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV4 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testCase.bucketName),
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV4, reqV4)
+		if recV4.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV4.Code)
+		}
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV2 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testCase.bucketName),
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV2, reqV2)
+		if recV2.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
+		}
+	}
+
+	// Test for Anonymous/unsigned http request.
+	// Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference.
+	bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)
+	// create unsigned HTTP request for PutBucketPolicyHandler.
+	anonReq, err := newTestRequest("PUT", getPutPolicyURL("", bucketName),
+		int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)))
+
+	if err != nil {
+		t.Fatalf("Minio %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
+			instanceType, bucketName, err)
+	}
+
+	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
+	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
+	// unsigned request goes through and its validated again.
+	ExecObjectLayerAPIAnonTest(t, "PutBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+
+	// HTTP request for testing when `objectLayer` is set to `nil`.
+	// There is no need to use an existing bucket and valid input for creating the request
+	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
+	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
+	nilBucket := "dummy-bucket"
+
+	nilReq, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", nilBucket),
+		0, nil, "", "")
+
+	if err != nil {
+		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
+	}
+	// execute the object layer set to `nil` test.
+	// `ExecObjectLayerAPINilTest` manages the operation.
+	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
+}
+
+// Wrapper for calling Get Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+func TestGetBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testGetBucketPolicyHandler, []string{"PutBucketPolicy", "GetBucketPolicy"})
+}
+
+// testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket.
+func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	// initialize bucket policy.
+	initBucketPolicies(obj)
+
+	// template for constructing HTTP request body for PUT bucket policy.
+	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"],"Sid":""}]}`
+
+	// Writing bucket policy before running test on GetBucketPolicy.
+	putTestPolicies := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, http.StatusNoContent},
+	}
+
+	// Iterating over the cases and writing the bucket policy.
+	// its required to write the policies first before running tests on GetBucketPolicy.
+	for i, testPolicy := range putTestPolicies {
+		// obtain the put bucket policy request body.
+		bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV4 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV4, reqV4)
+		if recV4.Code != testPolicy.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, recV4.Code)
+		}
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV2 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV2, reqV2)
+		if recV2.Code != testPolicy.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, recV2.Code)
+		}
+	}
+
+	// test cases with inputs and expected result for GetBucketPolicyHandler.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected output.
+		expectedBucketPolicy string
+		expectedRespStatus   int
+	}{
+		// Test case - 1.
+		// Case which valid inputs, expected to return success status of 200OK.
+		{
+			bucketName:           bucketName,
+			accessKey:            credentials.AccessKeyID,
+			secretKey:            credentials.SecretAccessKey,
+			expectedBucketPolicy: bucketPolicyTemplate,
+			expectedRespStatus:   http.StatusOK,
+		},
+		// Test case - 2.
+		// Case with non-existent bucket name.
+		{
+			bucketName:           "non-existent-bucket",
+			accessKey:            credentials.AccessKeyID,
+			secretKey:            credentials.SecretAccessKey,
+			expectedBucketPolicy: bucketPolicyTemplate,
+			expectedRespStatus:   http.StatusNotFound,
+		},
+		// Test case - 3.
+		// Case with invalid bucket name.
+		{
+			bucketName:           ".invalid-bucket-name",
+			accessKey:            credentials.AccessKeyID,
+			secretKey:            credentials.SecretAccessKey,
+			expectedBucketPolicy: "",
+			expectedRespStatus:   http.StatusBadRequest,
+		},
+	}
+	// Iterating over the cases, fetching the policy and validating the response.
+	for i, testCase := range testCases {
+		// expected bucket policy json string.
+		expectedBucketPolicyStr := fmt.Sprintf(testCase.expectedBucketPolicy, testCase.bucketName, testCase.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV4 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV4, err := newTestSignedRequestV4("GET", getGetPolicyURL("", testCase.bucketName),
+			0, nil, testCase.accessKey, testCase.secretKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler, GetBucketPolicyHandler handles the request.
+		apiRouter.ServeHTTP(recV4, reqV4)
+		// Assert the response code with the expected status.
+		if recV4.Code != testCase.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV4.Code)
+		}
+		// read the response body.
+		bucketPolicyReadBuf, err := ioutil.ReadAll(recV4.Body)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
+		}
+
+		if recV4.Code != testCase.expectedRespStatus {
+			// Verify whether the bucket policy fetched is same as the one inserted.
+			if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+				t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
+			}
+		}
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV2 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV2, err := newTestSignedRequestV2("GET", getGetPolicyURL("", testCase.bucketName),
+			0, nil, testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler, GetBucketPolicyHandler handles the request.
+		apiRouter.ServeHTTP(recV2, reqV2)
+		// Assert the response code with the expected status.
+		if recV2.Code != testCase.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV2.Code)
+		}
+		// read the response body.
+		bucketPolicyReadBuf, err = ioutil.ReadAll(recV2.Body)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
+		}
+		if recV2.Code == http.StatusOK {
+			// Verify whether the bucket policy fetched is same as the one inserted.
+			if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+				t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
+			}
+		}
+	}
+
+	// Test for Anonymous/unsigned http request.
+	// Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference.
+	// create unsigned HTTP request for PutBucketPolicyHandler.
+	anonReq, err := newTestRequest("GET", getPutPolicyURL("", bucketName), 0, nil)
+
+	if err != nil {
+		t.Fatalf("Minio %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
+			instanceType, bucketName, err)
+	}
+
+	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
+	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
+	// unsigned request goes through and its validated again.
+	ExecObjectLayerAPIAnonTest(t, "GetBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+
+	// HTTP request for testing when `objectLayer` is set to `nil`.
+	// There is no need to use an existing bucket and valid input for creating the request
+	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
+	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
+	nilBucket := "dummy-bucket"
+
+	nilReq, err := newTestSignedRequestV4("GET", getGetPolicyURL("", nilBucket),
+		0, nil, "", "")
+
+	if err != nil {
+		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
+	}
+	// execute the object layer set to `nil` test.
+	// `ExecObjectLayerAPINilTest` manages the operation.
+	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
+}
+
+// Wrapper for calling Delete Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+func TestDeleteBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testDeleteBucketPolicyHandler, []string{"PutBucketPolicy", "DeleteBucketPolicy"})
+}
+
+// testDeleteBucketPolicyHandler - Test for Delete bucket policy end point.
+func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
+	// initialize bucket policy.
+	initBucketPolicies(obj)
+
+	// template for constructing HTTP request body for PUT bucket policy.
+	bucketPolicyTemplate := `{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "s3:GetBucketLocation",
+                "s3:ListBucket"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s"
+            ]
+        },
+        {
+            "Action": [
+                "s3:GetObject"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s/this*"
+            ]
+        }
+    ]
+}`
+
+	// Writing bucket policy before running test on DeleteBucketPolicy.
+	putTestPolicies := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{
+			bucketName:         bucketName,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNoContent,
+		},
+	}
+
+	// Iterating over the cases and writing the bucket policy.
+	// its required to write the policies first before running tests on GetBucketPolicy.
+	for i, testPolicy := range putTestPolicies {
+		// obtain the put bucket policy request body.
+		bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV4 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV4, reqV4)
+		if recV4.Code != testPolicy.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, recV4.Code)
+		}
+	}
+
+	// testcases with input and expected output for DeleteBucketPolicyHandler.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected response.
+		expectedRespStatus int
+	}{
+		// Test case - 1.
+		{
+			bucketName:         bucketName,
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNoContent,
+		},
+		// Test case - 2.
+		// Case with non-existent-bucket.
+		{
+			bucketName:         "non-existent-bucket",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 3.
+		// Case with invalid bucket name.
+		{
+			bucketName:         ".invalid-bucket-name",
+			accessKey:          credentials.AccessKeyID,
+			secretKey:          credentials.SecretAccessKey,
+			expectedRespStatus: http.StatusBadRequest,
+		},
+	}
+	// Iterating over the cases and deleting the bucket policy and then asserting response.
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV4 := httptest.NewRecorder()
+		// construct HTTP request for Delete bucket policy endpoint.
+		reqV4, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", testCase.bucketName),
+			0, nil, testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler, DeleteBucketPolicyHandler  handles the request.
+		apiRouter.ServeHTTP(recV4, reqV4)
+		// Assert the response code with the expected status.
+		if recV4.Code != testCase.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV4.Code)
+		}
+	}
+
+	// Iterating over the cases and writing the bucket policy.
+	// its required to write the policies first before running tests on GetBucketPolicy.
+	for i, testPolicy := range putTestPolicies {
+		// obtain the put bucket policy request body.
+		bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV2 := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(recV2, reqV2)
+		if recV2.Code != testPolicy.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, recV2.Code)
+		}
+	}
+
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		recV2 := httptest.NewRecorder()
+		// construct HTTP request for Delete bucket policy endpoint.
+		reqV2, err := newTestSignedRequestV2("DELETE", getDeletePolicyURL("", testCase.bucketName),
+			0, nil, testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler, DeleteBucketPolicyHandler  handles the request.
+		apiRouter.ServeHTTP(recV2, reqV2)
+		// Assert the response code with the expected status.
+		if recV2.Code != testCase.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV2.Code)
+		}
+	}
+	// Test for Anonymous/unsigned http request.
+	// Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference.
+	// create unsigned HTTP request for PutBucketPolicyHandler.
+	anonReq, err := newTestRequest("DELETE", getPutPolicyURL("", bucketName), 0, nil)
+
+	if err != nil {
+		t.Fatalf("Minio %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
+			instanceType, bucketName, err)
+	}
+
+	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
+	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
+	// unsigned request goes through and its validated again.
+	ExecObjectLayerAPIAnonTest(t, "DeleteBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+
+	// HTTP request for testing when `objectLayer` is set to `nil`.
+	// There is no need to use an existing bucket and valid input for creating the request
+	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
+	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
+	nilBucket := "dummy-bucket"
+
+	nilReq, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", nilBucket),
+		0, nil, "", "")
+
+	if err != nil {
+		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
+	}
+	// execute the object layer set to `nil` test.
+	// `ExecObjectLayerAPINilTest` manages the operation.
+	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
+}
+
+// TestBucketPolicyConditionMatch - Tests to validate whether bucket policy conditions match.
+func TestBucketPolicyConditionMatch(t *testing.T) {
+	// obtain the inner map[string]set.StringSet for policyStatement.Conditions .
+	getInnerMap := func(key2, value string) map[string]set.StringSet {
+		innerMap := make(map[string]set.StringSet)
+		innerMap[key2] = set.CreateStringSet(value)
+		return innerMap
+	}
+
+	// obtain policyStatement with Conditions set.
+	getStatementWithCondition := func(key1, key2, value string) policyStatement {
+		innerMap := getInnerMap(key2, value)
+		// to set policyStatment.Conditions .
+		conditions := make(map[string]map[string]set.StringSet)
+		conditions[key1] = innerMap
+		// new policy statement.
+		statement := policyStatement{}
+		// set the condition.
+		statement.Conditions = conditions
+		return statement
+	}
+
+	testCases := []struct {
+		statementCondition policyStatement
+		condition          map[string]set.StringSet
+
+		expectedMatch bool
+	}{
+
+		// Test case - 1.
+		// StringEquals condition matches.
+		{
+
+			statementCondition: getStatementWithCondition("StringEquals", "s3:prefix", "Asia/"),
+			condition:          getInnerMap("prefix", "Asia/"),
+
+			expectedMatch: true,
+		},
+		// Test case - 2.
+		// StringEquals condition doesn't match.
+		{
+
+			statementCondition: getStatementWithCondition("StringEquals", "s3:prefix", "Asia/"),
+			condition:          getInnerMap("prefix", "Africa/"),
+
+			expectedMatch: false,
+		},
+		// Test case - 3.
+		// StringEquals condition matches.
+		{
+
+			statementCondition: getStatementWithCondition("StringEquals", "s3:max-keys", "Asia/"),
+			condition:          getInnerMap("max-keys", "Asia/"),
+
+			expectedMatch: true,
+		},
+		// Test case - 4.
+		// StringEquals condition doesn't match.
+		{
+
+			statementCondition: getStatementWithCondition("StringEquals", "s3:max-keys", "Asia/"),
+			condition:          getInnerMap("max-keys", "Africa/"),
+
+			expectedMatch: false,
+		},
+		// Test case - 5.
+		// StringNotEquals condition matches.
+		{
+
+			statementCondition: getStatementWithCondition("StringNotEquals", "s3:prefix", "Asia/"),
+			condition:          getInnerMap("prefix", "Asia/"),
+
+			expectedMatch: true,
+		},
+		// Test case - 6.
+		// StringNotEquals condition doesn't match.
+		{
+
+			statementCondition: getStatementWithCondition("StringNotEquals", "s3:prefix", "Asia/"),
+			condition:          getInnerMap("prefix", "Africa/"),
+
+			expectedMatch: false,
+		},
+		// Test case - 7.
+		// StringNotEquals condition matches.
+		{
+
+			statementCondition: getStatementWithCondition("StringNotEquals", "s3:max-keys", "Asia/"),
+			condition:          getInnerMap("max-keys", "Asia/"),
+
+			expectedMatch: true,
+		},
+		// Test case - 8.
+		// StringNotEquals condition doesn't match.
+		{
+
+			statementCondition: getStatementWithCondition("StringNotEquals", "s3:max-keys", "Asia/"),
+			condition:          getInnerMap("max-keys", "Africa/"),
+
+			expectedMatch: false,
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("Test case %d: Failed.", i+1), func(t *testing.T) {
+			// call the function under test and assert the result with the expected result.
+			doesMatch := bucketPolicyConditionMatch(tc.condition, tc.statementCondition)
+			if tc.expectedMatch != doesMatch {
+				t.Errorf("Expected the match to be `%v`; got `%v`.", tc.expectedMatch, doesMatch)
+			}
+		})
+	}
+}

cmd/bucket-policy-migrate.go

@@ -0,0 +1,85 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const policyJSON = "policy.json"
+
+func getBucketFromPolicyPath(oldPolicyPath string) string {
+	bucketPrefix, _ := filepath.Split(oldPolicyPath)
+	_, bucketName := filepath.Split(strings.TrimSuffix(bucketPrefix, slashSeparator))
+	return bucketName
+}
+
+func cleanupOldBucketPolicyConfigs() error {
+	// Get old bucket policy config directory.
+	oldBucketsConfigDir, err := getOldBucketsConfigPath()
+	fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy")
+
+	// Recursively remove configDir/buckets/ - old bucket policy config location.
+	// N B This is called only if all bucket policies were successfully migrated.
+	return os.RemoveAll(oldBucketsConfigDir)
+}
+
+func migrateBucketPolicyConfig(objAPI ObjectLayer) error {
+	// Get old bucket policy config directory.
+	oldBucketsConfigDir, err := getOldBucketsConfigPath()
+	fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy")
+
+	// Check if config directory holding bucket policy exists before
+	// migration.
+	_, err = os.Stat(oldBucketsConfigDir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	// WalkFunc that migrates access-policy.json to
+	// .minio.sys/buckets/bucketName/policy.json on all disks.
+	migrateBucketPolicy := func(policyPath string, fileInfo os.FileInfo, err error) error {
+		// policyFile - e.g /configDir/sample-bucket/access-policy.json
+		if err != nil {
+			return err
+		}
+		// Skip entries that aren't bucket policy files.
+		if fileInfo.Name() != "access-policy.json" {
+			return nil
+		}
+		// Get bucketName from old policy file path.
+		bucketName := getBucketFromPolicyPath(policyPath)
+		// Read bucket policy config from old location.
+		policyBytes, err := ioutil.ReadFile(policyPath)
+		fatalIf(err, "Unable to read bucket policy to migrate bucket policy", policyPath)
+		newPolicyPath := retainSlash(bucketConfigPrefix) + retainSlash(bucketName) + policyJSON
+		var metadata map[string]string
+		sha256sum := ""
+		// Erasure code the policy config to all the disks.
+		_, err = objAPI.PutObject(minioMetaBucket, newPolicyPath, int64(len(policyBytes)), bytes.NewReader(policyBytes), metadata, sha256sum)
+		fatalIf(err, "Unable to write bucket policy during migration.", newPolicyPath)
+		return nil
+	}
+	return filepath.Walk(oldBucketsConfigDir, migrateBucketPolicy)
+}

cmd/bucket-policy-parser.go

@@ -0,0 +1,348 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package cmd This file implements AWS Access Policy Language parser in
+// accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"github.com/minio/minio-go/pkg/set"
+)
+
+const (
+	// AWSResourcePrefix - bucket policy resource prefix.
+	AWSResourcePrefix = "arn:aws:s3:::"
+)
+
+// supportedActionMap - lists all the actions supported by minio.
+var supportedActionMap = set.CreateStringSet("*", "s3:*", "s3:GetObject",
+	"s3:ListBucket", "s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
+	"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts")
+
+// supported Conditions type.
+var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals")
+
+// Validate s3:prefix, s3:max-keys are present if not
+// supported keys for the conditions.
+var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys")
+
+// supportedEffectMap - supported effects.
+var supportedEffectMap = set.CreateStringSet("Allow", "Deny")
+
+// Statement - minio policy statement
+type policyStatement struct {
+	Actions    set.StringSet                       `json:"Action"`
+	Conditions map[string]map[string]set.StringSet `json:"Condition,omitempty"`
+	Effect     string
+	Principal  interface{}   `json:"Principal"`
+	Resources  set.StringSet `json:"Resource"`
+	Sid        string
+}
+
+// bucketPolicy - collection of various bucket policy statements.
+type bucketPolicy struct {
+	Version    string            // date in YYYY-MM-DD format
+	Statements []policyStatement `json:"Statement"`
+}
+
+// Stringer implementation for the bucket policies.
+func (b bucketPolicy) String() string {
+	bbytes, err := json.Marshal(&b)
+	if err != nil {
+		errorIf(err, "Unable to marshal bucket policy into JSON %#v", b)
+		return ""
+	}
+	return string(bbytes)
+}
+
+// isValidActions - are actions valid.
+func isValidActions(actions set.StringSet) (err error) {
+	// Statement actions cannot be empty.
+	if len(actions) == 0 {
+		err = errors.New("Action list cannot be empty")
+		return err
+	}
+	if unsupportedActions := actions.Difference(supportedActionMap); !unsupportedActions.IsEmpty() {
+		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document", unsupportedActions)
+		return err
+	}
+	return nil
+}
+
+// isValidEffect - is effect valid.
+func isValidEffect(effect string) (err error) {
+	// Statement effect cannot be empty.
+	if effect == "" {
+		err = errors.New("Policy effect cannot be empty")
+		return err
+	}
+	if !supportedEffectMap.Contains(effect) {
+		err = errors.New("Unsupported Effect found: ‘" + effect + "’, please validate your policy document")
+		return err
+	}
+	return nil
+}
+
+// isValidResources - are valid resources.
+func isValidResources(resources set.StringSet) (err error) {
+	// Statement resources cannot be empty.
+	if len(resources) == 0 {
+		err = errors.New("Resource list cannot be empty")
+		return err
+	}
+	for resource := range resources {
+		if !strings.HasPrefix(resource, AWSResourcePrefix) {
+			err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document")
+			return err
+		}
+		resourceSuffix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+		if len(resourceSuffix) == 0 || strings.HasPrefix(resourceSuffix, "/") {
+			err = errors.New("Invalid resource style found: ‘" + resource + "’, please validate your policy document")
+			return err
+		}
+	}
+	return nil
+}
+
+// Parse principals parses a incoming json. Handles cases for
+// these three combinations.
+// - "Principal": "*",
+// - "Principal": { "AWS" : "*" }
+// - "Principal": { "AWS" : [ "*" ]}
+func parsePrincipals(principal interface{}) set.StringSet {
+	principals, ok := principal.(map[string]interface{})
+	if !ok {
+		var principalStr string
+		principalStr, ok = principal.(string)
+		if ok {
+			return set.CreateStringSet(principalStr)
+		}
+	} // else {
+	var principalStrs []string
+	for _, p := range principals {
+		principalStr, isStr := p.(string)
+		if !isStr {
+			principalsAdd, isInterface := p.([]interface{})
+			if !isInterface {
+				principalStrsAddr, isStrs := p.([]string)
+				if !isStrs {
+					continue
+				}
+				principalStrs = append(principalStrs, principalStrsAddr...)
+			} else {
+				for _, pa := range principalsAdd {
+					var pstr string
+					pstr, isStr = pa.(string)
+					if !isStr {
+						continue
+					}
+					principalStrs = append(principalStrs, pstr)
+				}
+			}
+			continue
+		} // else {
+		principalStrs = append(principalStrs, principalStr)
+	}
+	return set.CreateStringSet(principalStrs...)
+}
+
+// isValidPrincipals - are valid principals.
+func isValidPrincipals(principal interface{}) (err error) {
+	principals := parsePrincipals(principal)
+	// Statement principal should have a value.
+	if len(principals) == 0 {
+		err = errors.New("Principal cannot be empty")
+		return err
+	}
+	if unsuppPrincipals := principals.Difference(set.CreateStringSet([]string{"*"}...)); !unsuppPrincipals.IsEmpty() {
+		// Minio does not support or implement IAM, "*" is the only valid value.
+		// Amazon s3 doc on principals: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
+		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document", unsuppPrincipals)
+		return err
+	}
+	return nil
+}
+
+// isValidConditions - are valid conditions.
+func isValidConditions(conditions map[string]map[string]set.StringSet) (err error) {
+	// Verify conditions should be valid.
+	// Validate if stringEquals, stringNotEquals are present
+	// if not throw an error.
+	conditionKeyVal := make(map[string]set.StringSet)
+	for conditionType := range conditions {
+		if !supportedConditionsType.Contains(conditionType) {
+			err = fmt.Errorf("Unsupported condition type '%s', please validate your policy document", conditionType)
+			return err
+		}
+		for key, value := range conditions[conditionType] {
+			if !supportedConditionsKey.Contains(key) {
+				err = fmt.Errorf("Unsupported condition key '%s', please validate your policy document", conditionType)
+				return err
+			}
+			conditionVal, ok := conditionKeyVal[key]
+			if ok && !value.Intersection(conditionVal).IsEmpty() {
+				err = fmt.Errorf("Ambigious condition values for key '%s', please validate your policy document", key)
+				return err
+			}
+			conditionKeyVal[key] = value
+		}
+	}
+	return nil
+}
+
+// List of actions for which prefixes are not allowed.
+var invalidPrefixActions = set.StringSet{
+	"s3:GetBucketLocation":          {},
+	"s3:ListBucket":                 {},
+	"s3:ListBucketMultipartUploads": {},
+	// Add actions which do not honor prefixes.
+}
+
+// resourcePrefix - provides the prefix removing any wildcards.
+func resourcePrefix(resource string) string {
+	if strings.HasSuffix(resource, "*") {
+		resource = strings.TrimSuffix(resource, "*")
+	}
+	return resource
+}
+
+// checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
+// First valation of Resources done for given set of Actions.
+// Later its validated for recursive Resources.
+func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIErrorCode {
+	// Validate statements for special actions and collect resources
+	// for others to validate nesting.
+	var resourceMap = set.NewStringSet()
+	for _, statement := range bucketPolicy.Statements {
+		for action := range statement.Actions {
+			for resource := range statement.Resources {
+				resourcePrefix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+				if _, ok := invalidPrefixActions[action]; ok {
+					// Resource prefix is not equal to bucket for
+					// prefix invalid actions, reject them.
+					if resourcePrefix != bucket {
+						return ErrMalformedPolicy
+					}
+				} else {
+					// For all other actions validate if resourcePrefix begins
+					// with bucket name, if not reject them.
+					if strings.Split(resourcePrefix, "/")[0] != bucket {
+						return ErrMalformedPolicy
+					}
+					// All valid resources collect them separately to verify nesting.
+					resourceMap.Add(resourcePrefix)
+				}
+			}
+		}
+	}
+
+	var resources []string
+	for resource := range resourceMap {
+		resources = append(resources, resourcePrefix(resource))
+	}
+
+	// Sort strings as shorter first.
+	sort.Strings(resources)
+
+	for len(resources) > 1 {
+		var resource string
+		resource, resources = resources[0], resources[1:]
+		// Loop through all resources, if one of them matches with
+		// previous shorter one, it means we have detected
+		// nesting. Reject such rules.
+		for _, otherResource := range resources {
+			// Common prefix reject such rules.
+			if strings.HasPrefix(otherResource, resource) {
+				return ErrPolicyNesting
+			}
+		}
+	}
+
+	// No errors found.
+	return ErrNone
+}
+
+// parseBucketPolicy - parses and validates if bucket policy is of
+// proper JSON and follows allowed restrictions with policy standards.
+func parseBucketPolicy(bucketPolicyReader io.Reader, policy *bucketPolicy) (err error) {
+	// Parse bucket policy reader.
+	decoder := json.NewDecoder(bucketPolicyReader)
+	if err = decoder.Decode(&policy); err != nil {
+		return err
+	}
+
+	// Policy version cannot be empty.
+	if len(policy.Version) == 0 {
+		err = errors.New("Policy version cannot be empty")
+		return err
+	}
+
+	// Policy statements cannot be empty.
+	if len(policy.Statements) == 0 {
+		err = errors.New("Policy statement cannot be empty")
+		return err
+	}
+
+	// Loop through all policy statements and validate entries.
+	for _, statement := range policy.Statements {
+		// Statement effect should be valid.
+		if err := isValidEffect(statement.Effect); err != nil {
+			return err
+		}
+		// Statement principal should be supported format.
+		if err := isValidPrincipals(statement.Principal); err != nil {
+			return err
+		}
+		// Statement actions should be valid.
+		if err := isValidActions(statement.Actions); err != nil {
+			return err
+		}
+		// Statement resources should be valid.
+		if err := isValidResources(statement.Resources); err != nil {
+			return err
+		}
+		// Statement conditions should be valid.
+		if err := isValidConditions(statement.Conditions); err != nil {
+			return err
+		}
+	}
+
+	// Separate deny and allow statements, so that we can apply deny
+	// statements in the beginning followed by Allow statements.
+	var denyStatements []policyStatement
+	var allowStatements []policyStatement
+	for _, statement := range policy.Statements {
+		if statement.Effect == "Deny" {
+			denyStatements = append(denyStatements, statement)
+			continue
+		}
+		// else if statement.Effect == "Allow"
+		allowStatements = append(allowStatements, statement)
+	}
+
+	// Deny statements are enforced first once matched.
+	policy.Statements = append(denyStatements, allowStatements...)
+
+	// Return successfully parsed policy structure.
+	return nil
+}

cmd/bucket-policy-parser_test.go

@@ -0,0 +1,711 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/minio/minio-go/pkg/set"
+)
+
+// Common bucket actions for both read and write policies.
+var (
+	readWriteBucketActions = []string{
+		"s3:GetBucketLocation",
+		"s3:ListBucket",
+		"s3:ListBucketMultipartUploads",
+		// Add more bucket level read-write actions here.
+	}
+	readWriteObjectActions = []string{
+		"s3:AbortMultipartUpload",
+		"s3:DeleteObject",
+		"s3:GetObject",
+		"s3:ListMultipartUploadParts",
+		"s3:PutObject",
+		// Add more object level read-write actions here.
+	}
+)
+
+// Write only actions.
+var (
+	writeOnlyBucketActions = []string{
+		"s3:GetBucketLocation",
+		"s3:ListBucketMultipartUploads",
+		// Add more bucket level write actions here.
+	}
+	writeOnlyObjectActions = []string{
+		"s3:AbortMultipartUpload",
+		"s3:DeleteObject",
+		"s3:ListMultipartUploadParts",
+		"s3:PutObject",
+		// Add more object level write actions here.
+	}
+)
+
+// Read only actions.
+var (
+	readOnlyBucketActions = []string{
+		"s3:GetBucketLocation",
+		"s3:ListBucket",
+		// Add more bucket level read actions here.
+	}
+	readOnlyObjectActions = []string{
+		"s3:GetObject",
+		// Add more object level read actions here.
+	}
+)
+
+// Obtain bucket statement for read-write bucketPolicy.
+func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
+	objectResourceStatement.Effect = "Allow"
+	objectResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
+	}
+	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...)
+	objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
+	return objectResourceStatement
+}
+
+// Obtain object statement for read-write bucketPolicy.
+func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatement {
+	bucketResourceStatement := policyStatement{}
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
+	}
+	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...)
+	bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
+	return bucketResourceStatement
+}
+
+// Obtain statements for read-write bucketPolicy.
+func getReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
+	// Save the read write policy.
+	statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
+	return statements
+}
+
+// Obtain bucket statement for read only bucketPolicy.
+func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
+	bucketResourceStatement := policyStatement{}
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
+	}
+	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...)
+	bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
+	return bucketResourceStatement
+}
+
+// Obtain object statement for read only bucketPolicy.
+func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
+	objectResourceStatement.Effect = "Allow"
+	objectResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
+	}
+	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...)
+	objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
+	return objectResourceStatement
+}
+
+// Obtain statements for read only bucketPolicy.
+func getReadOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
+	// Save the read only policy.
+	statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
+	return statements
+}
+
+// Obtain bucket statements for write only bucketPolicy.
+func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
+
+	bucketResourceStatement := policyStatement{}
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
+	}
+	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}...)
+	bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
+	return bucketResourceStatement
+}
+
+// Obtain object statements for write only bucketPolicy.
+func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
+	objectResourceStatement.Effect = "Allow"
+	objectResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
+	}
+	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}...)
+	objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
+	return objectResourceStatement
+}
+
+// Obtain statements for write only bucketPolicy.
+func getWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
+	// Write only policy.
+	// Save the write only policy.
+	statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
+	return statements
+}
+
+// Tests validate Action validator.
+func TestIsValidActions(t *testing.T) {
+	testCases := []struct {
+		// input.
+		actions set.StringSet
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Action.
+		// Test case - 1.
+		// "s3:ListObject" is an invalid Action.
+		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListObject", "s3:RemoveObject"}...),
+			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:RemoveObject\":struct {}{}, \"s3:ListObject\":struct {}{}}’, please validate your policy document"), false},
+		// Test case - 2.
+		// Empty Actions.
+		{set.CreateStringSet([]string{}...), errors.New("Action list cannot be empty"), false},
+		// Test case - 3.
+		// "s3:DeleteEverything"" is an invalid Action.
+		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...),
+			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document"), false},
+		// Inputs with valid Action.
+		// Test Case - 4.
+		{set.CreateStringSet([]string{
+			"s3:*", "*", "s3:GetObject", "s3:ListBucket",
+			"s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
+			"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads",
+			"s3:ListMultipartUploadParts"}...), nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidActions(testCase.actions)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+	}
+}
+
+// Tests validate Effect validator.
+func TestIsValidEffect(t *testing.T) {
+	testCases := []struct {
+		// input.
+		effect string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Effect.
+		// Test case - 1.
+		{"", errors.New("Policy effect cannot be empty"), false},
+		// Test case - 2.
+		{"DontAllow", errors.New("Unsupported Effect found: ‘DontAllow’, please validate your policy document"), false},
+		// Test case - 3.
+		{"NeverAllow", errors.New("Unsupported Effect found: ‘NeverAllow’, please validate your policy document"), false},
+		// Test case - 4.
+		{"AllowAlways", errors.New("Unsupported Effect found: ‘AllowAlways’, please validate your policy document"), false},
+
+		// Inputs with valid Effect.
+		// Test Case - 5.
+		{"Allow", nil, true},
+		// Test Case - 6.
+		{"Deny", nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidEffect(testCase.effect)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+	}
+}
+
+// Tests validate Resources validator.
+func TestIsValidResources(t *testing.T) {
+	testCases := []struct {
+		// input.
+		resources []string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Action.
+		// Test case - 1.
+		// Empty Resources.
+		{[]string{}, errors.New("Resource list cannot be empty"), false},
+		// Test case - 2.
+		// A valid resource should have prefix "arn:aws:s3:::".
+		{[]string{"my-resource"}, errors.New("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false},
+		// Test case - 3.
+		// A Valid resource should have bucket name followed by "arn:aws:s3:::".
+		{[]string{"arn:aws:s3:::"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document"), false},
+		// Test Case - 4.
+		// Valid resource shouldn't have slash('/') followed by "arn:aws:s3:::".
+		{[]string{"arn:aws:s3:::/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document"), false},
+
+		// Test cases with valid Resources.
+		{[]string{"arn:aws:s3:::my-bucket"}, nil, true},
+		{[]string{"arn:aws:s3:::my-bucket/Asia/*"}, nil, true},
+		{[]string{"arn:aws:s3:::my-bucket/Asia/India/*"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidResources(set.CreateStringSet(testCase.resources...))
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+	}
+}
+
+// Tests validate principals validator.
+func TestIsValidPrincipals(t *testing.T) {
+	testCases := []struct {
+		// input.
+		principals []string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Principals.
+		// Test case - 1.
+		// Empty Principals list.
+		{[]string{}, errors.New("Principal cannot be empty"), false},
+		// Test case - 2.
+		// "*" is the only valid principal.
+		{[]string{"my-principal"}, errors.New("Unsupported principals found: ‘set.StringSet{\"my-principal\":struct {}{}}’, please validate your policy document"), false},
+		// Test case - 3.
+		{[]string{"*", "111122233"}, errors.New("Unsupported principals found: ‘set.StringSet{\"111122233\":struct {}{}}’, please validate your policy document"), false},
+		// Test case - 4.
+		// Test case with valid principal value.
+		{[]string{"*"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidPrincipals(map[string]interface{}{
+			"AWS": testCase.principals,
+		})
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+	}
+}
+
+// Tests validate policyStatement condition validator.
+func TestIsValidConditions(t *testing.T) {
+	// returns empty conditions map.
+	setEmptyConditions := func() map[string]map[string]set.StringSet {
+		return make(map[string]map[string]set.StringSet)
+	}
+
+	// returns map with the "StringEquals" set to empty map.
+	setEmptyStringEquals := func() map[string]map[string]set.StringSet {
+		emptyMap := make(map[string]set.StringSet)
+		conditions := make(map[string]map[string]set.StringSet)
+		conditions["StringEquals"] = emptyMap
+		return conditions
+
+	}
+
+	// returns map with the "StringNotEquals" set to empty map.
+	setEmptyStringNotEquals := func() map[string]map[string]set.StringSet {
+		emptyMap := make(map[string]set.StringSet)
+		conditions := make(map[string]map[string]set.StringSet)
+		conditions["StringNotEquals"] = emptyMap
+		return conditions
+
+	}
+	// Generate conditions.
+	generateConditions := func(key1, key2, value string) map[string]map[string]set.StringSet {
+		innerMap := make(map[string]set.StringSet)
+		innerMap[key2] = set.CreateStringSet(value)
+		conditions := make(map[string]map[string]set.StringSet)
+		conditions[key1] = innerMap
+		return conditions
+	}
+
+	// generate ambigious conditions.
+	generateAmbigiousConditions := func() map[string]map[string]set.StringSet {
+		innerMap := make(map[string]set.StringSet)
+		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
+		conditions := make(map[string]map[string]set.StringSet)
+		conditions["StringEquals"] = innerMap
+		conditions["StringNotEquals"] = innerMap
+		return conditions
+	}
+
+	// generate valid and non valid type in the condition map.
+	generateValidInvalidConditions := func() map[string]map[string]set.StringSet {
+		innerMap := make(map[string]set.StringSet)
+		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
+		conditions := make(map[string]map[string]set.StringSet)
+		conditions["StringEquals"] = innerMap
+		conditions["InvalidType"] = innerMap
+		return conditions
+	}
+
+	// generate valid and invalid keys for valid types in the same condition map.
+	generateValidInvalidConditionKeys := func() map[string]map[string]set.StringSet {
+		innerMapValid := make(map[string]set.StringSet)
+		innerMapValid["s3:prefix"] = set.CreateStringSet("Asia/")
+		innerMapInValid := make(map[string]set.StringSet)
+		innerMapInValid["s3:invalid"] = set.CreateStringSet("Asia/")
+		conditions := make(map[string]map[string]set.StringSet)
+		conditions["StringEquals"] = innerMapValid
+		conditions["StringEquals"] = innerMapInValid
+		return conditions
+	}
+
+	// List of Conditions used for test cases.
+	testConditions := []map[string]map[string]set.StringSet{
+		generateConditions("StringValues", "s3:max-keys", "100"),
+		generateConditions("StringEquals", "s3:Object", "100"),
+		generateAmbigiousConditions(),
+		generateValidInvalidConditions(),
+		generateValidInvalidConditionKeys(),
+		setEmptyConditions(),
+		setEmptyStringEquals(),
+		setEmptyStringNotEquals(),
+		generateConditions("StringEquals", "s3:prefix", "Asia/"),
+		generateConditions("StringEquals", "s3:max-keys", "100"),
+		generateConditions("StringNotEquals", "s3:prefix", "Asia/"),
+		generateConditions("StringNotEquals", "s3:max-keys", "100"),
+	}
+
+	testCases := []struct {
+		inputCondition map[string]map[string]set.StringSet
+		// expected result.
+		expectedErr error
+		// flag indicating whether test should pass.
+		shouldPass bool
+	}{
+		// Malformed conditions.
+		// Test case - 1.
+		// "StringValues" is an invalid type.
+		{testConditions[0], fmt.Errorf("Unsupported condition type 'StringValues', " +
+			"please validate your policy document"), false},
+		// Test case - 2.
+		// "s3:Object" is an invalid key.
+		{testConditions[1], fmt.Errorf("Unsupported condition key " +
+			"'StringEquals', please validate your policy document"), false},
+		// Test case - 3.
+		// Test case with Ambigious conditions set.
+		{testConditions[2], fmt.Errorf("Ambigious condition values for key 's3:prefix', " +
+			"please validate your policy document"), false},
+		// Test case - 4.
+		// Test case with valid and invalid condition types.
+		{testConditions[3], fmt.Errorf("Unsupported condition type 'InvalidType', " +
+			"please validate your policy document"), false},
+		// Test case - 5.
+		// Test case with valid and invalid condition keys.
+		{testConditions[4], fmt.Errorf("Unsupported condition key 'StringEquals', " +
+			"please validate your policy document"), false},
+		// Test cases with valid conditions.
+		// Test case - 6.
+		{testConditions[5], nil, true},
+		// Test case - 7.
+		{testConditions[6], nil, true},
+		// Test case - 8.
+		{testConditions[7], nil, true},
+		// Test case - 9.
+		{testConditions[8], nil, true},
+		// Test case - 10.
+		{testConditions[9], nil, true},
+		// Test case - 11.
+		{testConditions[10], nil, true},
+		// Test case 10.
+		{testConditions[11], nil, true},
+	}
+	for i, testCase := range testCases {
+		actualErr := isValidConditions(testCase.inputCondition)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, actualErr.Error())
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.expectedErr.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if actualErr.Error() != testCase.expectedErr.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.expectedErr.Error(), actualErr.Error())
+			}
+		}
+	}
+}
+
+// Tests validate Policy Action and Resource fields.
+func TestCheckbucketPolicyResources(t *testing.T) {
+	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
+	setValidPrefixActions := func(statements []policyStatement) []policyStatement {
+		statements[0].Actions = set.CreateStringSet([]string{"s3:DeleteObject", "s3:PutObject"}...)
+		return statements
+	}
+	// contracting policy statement with recursive resources.
+	// should result in ErrMalformedPolicy
+	setRecurseResource := func(statements []policyStatement) []policyStatement {
+		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}...)
+		return statements
+	}
+
+	// constructing policy statement with lexically close characters.
+	// should not result in ErrMalformedPolicy
+	setResourceLexical := func(statements []policyStatement) []policyStatement {
+		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}...)
+		return statements
+	}
+
+	// List of bucketPolicy used for tests.
+	bucketAccessPolicies := []bucketPolicy{
+		// bucketPolicy - 1.
+		// Contains valid read only policy statement.
+		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
+		// bucketPolicy - 2.
+		// Contains valid read-write only policy statement.
+		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
+		// bucketPolicy - 3.
+		// Contains valid write only policy statement.
+		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
+		// bucketPolicy - 4.
+		// Contains invalidPrefixActions.
+		// Since resourcePrefix is not to the bucket-name, it return ErrMalformedPolicy.
+		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket-fail", "Asia/India/")},
+		// bucketPolicy - 5.
+		// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
+		// but bucket part of the resource is not equal to the bucket name.
+		// this results in return of ErrMalformedPolicy.
+		{Version: "1.0", Statements: setValidPrefixActions(getWriteOnlyStatement("minio-bucket-fail", "Asia/India/"))},
+		// bucketPolicy - 6.
+		// contracting policy statement with recursive resources.
+		// should result in ErrMalformedPolicy
+		{Version: "1.0", Statements: setRecurseResource(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "")))},
+		// BucketPolciy - 7.
+		// constructing policy statement with non recursive but
+		// lexically close resources.
+		// should result in ErrNone.
+		{Version: "1.0", Statements: setResourceLexical(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "oo")))},
+	}
+
+	testCases := []struct {
+		inputPolicy bucketPolicy
+		// expected results.
+		apiErrCode APIErrorCode
+		// Flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Test case - 1.
+		{bucketAccessPolicies[0], ErrNone, true},
+		// Test case - 2.
+		{bucketAccessPolicies[1], ErrNone, true},
+		// Test case - 3.
+		{bucketAccessPolicies[2], ErrNone, true},
+		// Test case - 4.
+		// contains invalidPrefixActions (check bucket-policy-parser.go).
+		// Resource prefix will not be equal to the bucket name in this case.
+		{bucketAccessPolicies[3], ErrMalformedPolicy, false},
+		// Test case - 5.
+		// actions contain invalidPrefixActions (check bucket-policy-parser.go).
+		// Resource prefix bucket part is not equal to the bucket name in this case.
+		{bucketAccessPolicies[4], ErrMalformedPolicy, false},
+		// Test case - 6.
+		// contracting policy statement with recursive resources.
+		// should result in ErrPolicyNesting.
+		{bucketAccessPolicies[5], ErrPolicyNesting, false},
+		// Test case - 7.
+		// constructing policy statement with lexically close
+		// characters.
+		// should result in ErrNone.
+		{bucketAccessPolicies[6], ErrNone, true},
+	}
+	for i, testCase := range testCases {
+		apiErrCode := checkBucketPolicyResources("minio-bucket", &testCase.inputPolicy)
+		if apiErrCode != ErrNone && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
+		}
+		if apiErrCode == ErrNone && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with ErrCode %v, but passed instead", i+1, testCase.apiErrCode)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if apiErrCode != ErrNone && !testCase.shouldPass {
+			if testCase.apiErrCode != apiErrCode {
+				t.Errorf("Test %d: Expected to fail with error code %v, but instead failed with error code %v", i+1, testCase.apiErrCode, apiErrCode)
+			}
+		}
+	}
+}
+
+// Tests validate parsing of BucketAccessPolicy.
+func TestParseBucketPolicy(t *testing.T) {
+	// set Unsupported Actions.
+	setUnsupportedActions := func(statements []policyStatement) []policyStatement {
+		// "s3:DeleteEverything"" is an Unsupported Action.
+		statements[0].Actions = set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...)
+		return statements
+	}
+	// set unsupported Effect.
+	setUnsupportedEffect := func(statements []policyStatement) []policyStatement {
+		// Effect "Don't allow" is Unsupported.
+		statements[0].Effect = "DontAllow"
+		return statements
+	}
+	// set unsupported principals.
+	setUnsupportedPrincipals := func(statements []policyStatement) []policyStatement {
+		// "User1111"" is an Unsupported Principal.
+		statements[0].Principal = map[string]interface{}{
+			"AWS": []string{"*", "User1111"},
+		}
+		return statements
+	}
+	// set unsupported Resources.
+	setUnsupportedResources := func(statements []policyStatement) []policyStatement {
+		// "s3:DeleteEverything"" is an Unsupported Action.
+		statements[0].Resources = set.CreateStringSet([]string{"my-resource"}...)
+		return statements
+	}
+	// List of bucketPolicy used for test cases.
+	bucketAccesPolicies := []bucketPolicy{
+		// bucketPolicy - 0.
+		// bucketPolicy statement empty.
+		{Version: "1.0"},
+		// bucketPolicy - 1.
+		// bucketPolicy version empty.
+		{Version: "", Statements: []policyStatement{}},
+		// bucketPolicy - 2.
+		// Readonly bucketPolicy.
+		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
+		// bucketPolicy - 3.
+		// Read-Write bucket policy.
+		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
+		// bucketPolicy - 4.
+		// Write only bucket policy.
+		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
+		// bucketPolicy - 5.
+		// bucketPolicy statement contains unsupported action.
+		{Version: "1.0", Statements: setUnsupportedActions(getReadOnlyStatement("minio-bucket", ""))},
+		// bucketPolicy - 6.
+		// bucketPolicy statement contains unsupported Effect.
+		{Version: "1.0", Statements: setUnsupportedEffect(getReadWriteStatement("minio-bucket", "Asia/"))},
+		// bucketPolicy - 7.
+		// bucketPolicy statement contains unsupported Principal.
+		{Version: "1.0", Statements: setUnsupportedPrincipals(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
+		// bucketPolicy - 8.
+		// bucketPolicy statement contains unsupported Resource.
+		{Version: "1.0", Statements: setUnsupportedResources(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
+	}
+
+	testCases := []struct {
+		inputPolicy bucketPolicy
+		// expected results.
+		expectedPolicy bucketPolicy
+		err            error
+		// Flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Test case - 1.
+		// bucketPolicy statement empty.
+		{bucketAccesPolicies[0], bucketPolicy{}, errors.New("Policy statement cannot be empty"), false},
+		// Test case - 2.
+		// bucketPolicy version empty.
+		{bucketAccesPolicies[1], bucketPolicy{}, errors.New("Policy version cannot be empty"), false},
+		// Test case - 3.
+		// Readonly bucketPolicy.
+		{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
+		// Test case - 4.
+		// Read-Write bucket policy.
+		{bucketAccesPolicies[3], bucketAccesPolicies[3], nil, true},
+		// Test case - 5.
+		// Write only bucket policy.
+		{bucketAccesPolicies[4], bucketAccesPolicies[4], nil, true},
+		// Test case - 6.
+		// bucketPolicy statement contains unsupported action.
+		{bucketAccesPolicies[5], bucketAccesPolicies[5], fmt.Errorf("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document"), false},
+		// Test case - 7.
+		// bucketPolicy statement contains unsupported Effect.
+		{bucketAccesPolicies[6], bucketAccesPolicies[6], fmt.Errorf("Unsupported Effect found: ‘DontAllow’, please validate your policy document"), false},
+		// Test case - 8.
+		// bucketPolicy statement contains unsupported Principal.
+		{bucketAccesPolicies[7], bucketAccesPolicies[7], fmt.Errorf("Unsupported principals found: ‘set.StringSet{\"User1111\":struct {}{}}’, please validate your policy document"), false},
+		// Test case - 9.
+		// bucketPolicy statement contains unsupported Resource.
+		{bucketAccesPolicies[8], bucketAccesPolicies[8], fmt.Errorf("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false},
+	}
+	for i, testCase := range testCases {
+		var buffer bytes.Buffer
+		encoder := json.NewEncoder(&buffer)
+		err := encoder.Encode(testCase.inputPolicy)
+		if err != nil {
+			t.Fatalf("Test %d: Couldn't Marshal bucket policy %s", i+1, err)
+		}
+
+		var actualAccessPolicy = &bucketPolicy{}
+		err = parseBucketPolicy(&buffer, actualAccessPolicy)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.expectedPolicy.String() != actualAccessPolicy.String() {
+				t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
+			}
+		}
+	}
+}

cmd/bucket-policy.go

@@ -0,0 +1,286 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"path"
+	"sync"
+)
+
+// Variable represents bucket policies in memory.
+var globalBucketPolicies *bucketPolicies
+
+// Global bucket policies list, policies are enforced on each bucket looking
+// through the policies here.
+type bucketPolicies struct {
+	rwMutex *sync.RWMutex
+
+	// Collection of 'bucket' policies.
+	bucketPolicyConfigs map[string]*bucketPolicy
+}
+
+// Represent a policy change
+type policyChange struct {
+	// isRemove is true if the policy change is to delete the
+	// policy on a bucket.
+	IsRemove bool
+
+	// represents the new policy for the bucket
+	BktPolicy *bucketPolicy
+}
+
+// Fetch bucket policy for a given bucket.
+func (bp bucketPolicies) GetBucketPolicy(bucket string) *bucketPolicy {
+	bp.rwMutex.RLock()
+	defer bp.rwMutex.RUnlock()
+	return bp.bucketPolicyConfigs[bucket]
+}
+
+// Set a new bucket policy for a bucket, this operation will overwrite
+// any previous bucket policies for the bucket.
+func (bp *bucketPolicies) SetBucketPolicy(bucket string, pCh policyChange) error {
+	bp.rwMutex.Lock()
+	defer bp.rwMutex.Unlock()
+
+	if pCh.IsRemove {
+		delete(bp.bucketPolicyConfigs, bucket)
+	} else {
+		if pCh.BktPolicy == nil {
+			return errInvalidArgument
+		}
+		bp.bucketPolicyConfigs[bucket] = pCh.BktPolicy
+	}
+	return nil
+}
+
+// Loads all bucket policies from persistent layer.
+func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolicy, err error) {
+	// List buckets to proceed loading all notification configuration.
+	buckets, err := objAPI.ListBuckets()
+	errorIf(err, "Unable to list buckets.")
+	if err != nil {
+		return nil, errorCause(err)
+	}
+
+	policies = make(map[string]*bucketPolicy)
+	var pErrs []error
+	// Loads bucket policy.
+	for _, bucket := range buckets {
+		policy, pErr := readBucketPolicy(bucket.Name, objAPI)
+		if pErr != nil {
+			// net.Dial fails for rpc client or any
+			// other unexpected errors during net.Dial.
+			if !isErrIgnored(pErr, errDiskNotFound) {
+				if !isErrBucketPolicyNotFound(pErr) {
+					pErrs = append(pErrs, pErr)
+				}
+			}
+			// Continue to load other bucket policies if possible.
+			continue
+		}
+		policies[bucket.Name] = policy
+	}
+
+	// Look for any errors occurred while reading bucket policies.
+	for _, pErr := range pErrs {
+		if pErr != nil {
+			return policies, pErr
+		}
+	}
+
+	// Success.
+	return policies, nil
+}
+
+// Intialize all bucket policies.
+func initBucketPolicies(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+
+	// Read all bucket policies.
+	policies, err := loadAllBucketPolicies(objAPI)
+	if err != nil {
+		return err
+	}
+
+	// Populate global bucket collection.
+	globalBucketPolicies = &bucketPolicies{
+		rwMutex:             &sync.RWMutex{},
+		bucketPolicyConfigs: policies,
+	}
+
+	// Success.
+	return nil
+}
+
+// getOldBucketsConfigPath - get old buckets config path. (Only used for migrating old bucket policies)
+func getOldBucketsConfigPath() (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	return path.Join(configPath, "buckets"), nil
+}
+
+// readBucketPolicyJSON - reads bucket policy for an input bucket, returns BucketPolicyNotFound
+// if bucket policy is not found.
+func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader io.Reader, err error) {
+	policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
+
+	// Acquire a read lock on policy config before reading.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
+	objLock.RLock()
+	defer objLock.RUnlock()
+
+	objInfo, err := objAPI.GetObjectInfo(minioMetaBucket, policyPath)
+	if err != nil {
+		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
+			return nil, BucketPolicyNotFound{Bucket: bucket}
+		}
+		errorIf(err, "Unable to load policy for the bucket %s.", bucket)
+		return nil, errorCause(err)
+	}
+	var buffer bytes.Buffer
+	err = objAPI.GetObject(minioMetaBucket, policyPath, 0, objInfo.Size, &buffer)
+	if err != nil {
+		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
+			return nil, BucketPolicyNotFound{Bucket: bucket}
+		}
+		errorIf(err, "Unable to load policy for the bucket %s.", bucket)
+		return nil, errorCause(err)
+	}
+
+	return &buffer, nil
+}
+
+// readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
+// if bucket policy is not found. This function also parses the bucket policy into an object.
+func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
+	// Read bucket policy JSON.
+	bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse the saved policy.
+	var policy = &bucketPolicy{}
+	err = parseBucketPolicy(bucketPolicyReader, policy)
+	if err != nil {
+		return nil, err
+
+	}
+	return policy, nil
+}
+
+// removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound
+// if no policies are found.
+func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
+	policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
+	// Acquire a write lock on policy config before modifying.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
+	objLock.Lock()
+	defer objLock.Unlock()
+	if err := objAPI.DeleteObject(minioMetaBucket, policyPath); err != nil {
+		errorIf(err, "Unable to remove bucket-policy on bucket %s.", bucket)
+		err = errorCause(err)
+		if _, ok := err.(ObjectNotFound); ok {
+			return BucketPolicyNotFound{Bucket: bucket}
+		}
+		return err
+	}
+	return nil
+}
+
+// writeBucketPolicy - save a bucket policy that is assumed to be validated.
+func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy *bucketPolicy) error {
+	buf, err := json.Marshal(bpy)
+	if err != nil {
+		errorIf(err, "Unable to marshal bucket policy '%v' to JSON", *bpy)
+		return err
+	}
+	policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
+	// Acquire a write lock on policy config before modifying.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
+	objLock.Lock()
+	defer objLock.Unlock()
+	if _, err := objAPI.PutObject(minioMetaBucket, policyPath, int64(len(buf)), bytes.NewReader(buf), nil, ""); err != nil {
+		errorIf(err, "Unable to set policy for the bucket %s", bucket)
+		return errorCause(err)
+	}
+	return nil
+}
+
+func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI ObjectLayer) APIErrorCode {
+	// Parse bucket policy.
+	var policy = &bucketPolicy{}
+	err := parseBucketPolicy(bytes.NewReader(policyBytes), policy)
+	if err != nil {
+		errorIf(err, "Unable to parse bucket policy.")
+		return ErrInvalidPolicyDocument
+	}
+
+	// Parse check bucket policy.
+	if s3Error := checkBucketPolicyResources(bucket, policy); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Acquire a write lock on bucket before modifying its configuration.
+	bucketLock := globalNSMutex.NewNSLock(bucket, "")
+	bucketLock.Lock()
+	// Release lock after notifying peers
+	defer bucketLock.Unlock()
+
+	// Save bucket policy.
+	if err = persistAndNotifyBucketPolicyChange(bucket, policyChange{false, policy}, objAPI); err != nil {
+		switch err.(type) {
+		case BucketNameInvalid:
+			return ErrInvalidBucketName
+		case BucketNotFound:
+			return ErrNoSuchBucket
+		default:
+			errorIf(err, "Unable to save bucket policy.")
+			return ErrInternalError
+		}
+	}
+	return ErrNone
+}
+
+// persistAndNotifyBucketPolicyChange - takes a policyChange argument,
+// persists it to storage, and notify nodes in the cluster about the
+// change. In-memory state is updated in response to the notification.
+func persistAndNotifyBucketPolicyChange(bucket string, pCh policyChange, objAPI ObjectLayer) error {
+	if pCh.IsRemove {
+		if err := removeBucketPolicy(bucket, objAPI); err != nil {
+			return err
+		}
+	} else {
+		if pCh.BktPolicy == nil {
+			return errInvalidArgument
+		}
+		if err := writeBucketPolicy(bucket, objAPI, pCh.BktPolicy); err != nil {
+			return err
+		}
+	}
+
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketPolicy(bucket, pCh)
+	return nil
+}

cmd/build-constants.go

@@ -0,0 +1,33 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
+// set through ‘buildscripts/gen-ldflags.go’.
+var (
+	// GOPATH - GOPATH value at the time of build.
+	GOPATH = ""
+
+	// Version - version time.RFC3339.
+	Version = "DEVELOPMENT.GOGET"
+	// ReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ.
+	ReleaseTag = "DEVELOPMENT.GOGET"
+	// CommitID - latest commit id.
+	CommitID = "DEVELOPMENT.GOGET"
+	// ShortCommitID - first 12 characters from CommitID.
+	ShortCommitID = CommitID[:12]
+)

cmd/certs.go

@@ -0,0 +1,147 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// createCertsPath create certs path.
+func createCertsPath() error {
+	certsPath, err := getCertsPath()
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(certsPath, 0700); err != nil {
+		return err
+	}
+	rootCAsPath := filepath.Join(certsPath, globalMinioCertsCADir)
+	return os.MkdirAll(rootCAsPath, 0700)
+}
+
+// getCertsPath get certs path.
+func getCertsPath() (string, error) {
+	var certsPath string
+	configDir, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	certsPath = filepath.Join(configDir, globalMinioCertsDir)
+	return certsPath, nil
+}
+
+// mustGetCertsPath must get certs path.
+func mustGetCertsPath() string {
+	certsPath, err := getCertsPath()
+	fatalIf(err, "Failed to get certificate path.")
+	return certsPath
+}
+
+// mustGetCertFile must get cert file.
+func mustGetCertFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioCertFile)
+}
+
+// mustGetKeyFile must get key file.
+func mustGetKeyFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioKeyFile)
+}
+
+// mustGetCAFiles must get the list of the CA certificates stored in minio config dir
+func mustGetCAFiles() (caCerts []string) {
+	CAsDir := filepath.Join(mustGetCertsPath(), globalMinioCertsCADir)
+	caFiles, _ := ioutil.ReadDir(CAsDir)
+	for _, cert := range caFiles {
+		caCerts = append(caCerts, filepath.Join(CAsDir, cert.Name()))
+	}
+	return
+}
+
+// mustGetSystemCertPool returns empty cert pool in case of error (windows)
+func mustGetSystemCertPool() *x509.CertPool {
+	pool, err := x509.SystemCertPool()
+	if err != nil {
+		return x509.NewCertPool()
+	}
+	return pool
+}
+
+// isCertFileExists verifies if cert file exists, returns true if
+// found, false otherwise.
+func isCertFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioCertFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isKeyFileExists verifies if key file exists, returns true if found,
+// false otherwise.
+func isKeyFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioKeyFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isSSL - returns true with both cert and key exists.
+func isSSL() bool {
+	return isCertFileExists() && isKeyFileExists()
+}
+
+// Reads certificated file and returns a list of parsed certificates.
+func readCertificateChain() ([]*x509.Certificate, error) {
+	bytes, err := ioutil.ReadFile(mustGetCertFile())
+	if err != nil {
+		return nil, err
+	}
+
+	// Proceed to parse the certificates.
+	return parseCertificateChain(bytes)
+}
+
+// Parses certificate chain, returns a list of parsed certificates.
+func parseCertificateChain(bytes []byte) ([]*x509.Certificate, error) {
+	var certs []*x509.Certificate
+	var block *pem.Block
+	current := bytes
+
+	// Parse all certs in the chain.
+	for len(current) > 0 {
+		block, current = pem.Decode(current)
+		if block == nil {
+			return nil, errors.New("Could not PEM block")
+		}
+		// Parse the decoded certificate.
+		cert, err := x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			return nil, err
+		}
+		certs = append(certs, cert)
+
+	}
+	return certs, nil
+}

cmd/certs_test.go

@@ -0,0 +1,111 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Make sure we have a valid certs path.
+func TestGetCertsPath(t *testing.T) {
+	path, err := getCertsPath()
+	if err != nil {
+		t.Error(err)
+	}
+	if path == "" {
+		t.Errorf("expected path to not be an empty string, got: '%s'", path)
+	}
+	// Ensure it contains some sort of path separator.
+	if !strings.ContainsRune(path, os.PathSeparator) {
+		t.Errorf("expected path to contain file separator")
+	}
+	// It should also be an absolute path.
+	if !filepath.IsAbs(path) {
+		t.Errorf("expected path to be an absolute path")
+	}
+
+	// This will error if something goes wrong, so just call it.
+	mustGetCertsPath()
+}
+
+// Ensure that the certificate and key file getters contain their respective
+// file name and endings.
+func TestGetFiles(t *testing.T) {
+	file := mustGetCertFile()
+	if !strings.Contains(file, globalMinioCertFile) {
+		t.Errorf("CertFile does not contain %s", globalMinioCertFile)
+	}
+
+	file = mustGetKeyFile()
+	if !strings.Contains(file, globalMinioKeyFile) {
+		t.Errorf("KeyFile does not contain %s", globalMinioKeyFile)
+	}
+}
+
+// Parses .crt file contents
+func TestParseCertificateChain(t *testing.T) {
+	// given
+	cert := `-----BEGIN CERTIFICATE-----
+MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa
+WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN
+aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN
+AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0
+MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50
+ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF
+TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3
+DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83
+HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4
+aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU
+JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe
+nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK
+FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE
+M9ofSEt/bdRD
+-----END CERTIFICATE-----`
+
+	// when
+	certs, err := parseCertificateChain([]byte(cert))
+
+	// then
+	if err != nil {
+		t.Fatalf("Could not parse certificate: %s", err)
+	}
+
+	if len(certs) != 1 {
+		t.Fatalf("Expected number of certificates in chain was 1, actual: %d", len(certs))
+	}
+
+	if certs[0].Subject.CommonName != "Minio" {
+		t.Fatalf("Expected Subject.CommonName was Minio, actual: %s", certs[0].Subject.CommonName)
+	}
+}
+
+// Parses invalid .crt file contents and returns error
+func TestParseInvalidCertificateChain(t *testing.T) {
+	// given
+	cert := `This is now valid certificate`
+
+	// when
+	_, err := parseCertificateChain([]byte(cert))
+
+	// then
+	if err == nil {
+		t.Fatalf("Expected error but none occurred")
+	}
+}

cmd/checkport.go

@@ -0,0 +1,76 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net"
+	"os"
+	"syscall"
+)
+
+// Make sure that none of the other processes are listening on the
+// specified port on any of the interfaces.
+//
+// On linux if a process is listening on 127.0.0.1:9000 then Listen()
+// on ":9000" fails with the error "port already in use".
+// However on Mac OSX Listen() on ":9000" falls back to the IPv6 address.
+// This causes confusion on Mac OSX that minio server is not reachable
+// on 127.0.0.1 even though minio server is running. So before we start
+// the minio server we make sure that the port is free on each tcp network.
+//
+// Port is string on purpose here.
+// https://github.com/golang/go/issues/16142#issuecomment-245912773
+//
+// "Keep in mind that ports in Go are strings: https://play.golang.org/p/zk2WEri_E9"
+//                 - @bradfitz
+func checkPortAvailability(portStr string) error {
+	network := [3]string{"tcp", "tcp4", "tcp6"}
+	for _, n := range network {
+		l, err := net.Listen(n, net.JoinHostPort("", portStr))
+		if err != nil {
+			if isAddrInUse(err) {
+				// Return error if another process is listening on the
+				// same port.
+				return err
+			}
+			// Ignore any other error (ex. EAFNOSUPPORT)
+			continue
+		}
+
+		// look for error so we don't have dangling connection
+		if err = l.Close(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Return true if err is "address already in use" error.
+// syscall.EADDRINUSE is available on all OSes.
+func isAddrInUse(err error) bool {
+	if opErr, ok := err.(*net.OpError); ok {
+		if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
+			if errno, ok := sysErr.Err.(syscall.Errno); ok {
+				if errno == syscall.EADDRINUSE {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

cmd/checkport_test.go

@@ -0,0 +1,54 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net"
+	"runtime"
+	"testing"
+)
+
+// Tests for port availability logic written for server startup sequence.
+func TestCheckPortAvailability(t *testing.T) {
+	tests := []struct {
+		port string
+	}{
+		{getFreePort()},
+		{getFreePort()},
+	}
+	for _, test := range tests {
+		// This test should pass if the ports are available
+		err := checkPortAvailability(test.port)
+		if err != nil {
+			t.Fatalf("checkPortAvailability test failed for port: %s. Error: %v", test.port, err)
+		}
+
+		// Now use the ports and check again
+		ln, err := net.Listen("tcp", net.JoinHostPort("", test.port))
+		if err != nil {
+			t.Fail()
+		}
+		defer ln.Close()
+
+		err = checkPortAvailability(test.port)
+
+		// Skip if the os is windows due to https://github.com/golang/go/issues/7598
+		if err == nil && runtime.GOOS != "windows" {
+			t.Fatalf("checkPortAvailability should fail for port: %s. Error: %v", test.port, err)
+		}
+	}
+}

cmd/commands.go

@@ -14,17 +14,17 @@
  * limitations under the License.
  */
 
-package main
+package cmd
 
 import "github.com/minio/cli"
 
-// Collection of minio commands currently supported are
+// Collection of minio commands currently supported are.
 var commands = []cli.Command{}
 
-// Collection of minio commands currently supported in a trie tree
+// Collection of minio commands currently supported in a trie tree.
 var commandsTree = newTrie()
 
-// registerCommand registers a cli command
+// registerCommand registers a cli command.
 func registerCommand(command cli.Command) {
 	commands = append(commands, command)
 	commandsTree.Insert(command.Name)

cmd/config-migrate.go

@@ -0,0 +1,635 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/quick"
+)
+
+func migrateConfig() error {
+	// Purge all configs with version '1'.
+	if err := purgeV1(); err != nil {
+		return err
+	}
+	// Migrate version '2' to '3'.
+	if err := migrateV2ToV3(); err != nil {
+		return err
+	}
+	// Migrate version '3' to '4'.
+	if err := migrateV3ToV4(); err != nil {
+		return err
+	}
+	// Migrate version '4' to '5'.
+	if err := migrateV4ToV5(); err != nil {
+		return err
+	}
+	// Migrate version '5' to '6.
+	if err := migrateV5ToV6(); err != nil {
+		return err
+	}
+	// Migrate version '6' to '7'.
+	if err := migrateV6ToV7(); err != nil {
+		return err
+	}
+	// Migrate version '7' to '8'.
+	if err := migrateV7ToV8(); err != nil {
+		return err
+	}
+	// Migrate version '8' to '9'.
+	if err := migrateV8ToV9(); err != nil {
+		return err
+	}
+	// Migrate version '9' to '10'.
+	if err := migrateV9ToV10(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Version '1' is not supported anymore and deprecated, safe to delete.
+func purgeV1() error {
+	cv1, err := loadConfigV1()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
+
+	}
+
+	if cv1.Version == "1" {
+		console.Println("Removed unsupported config version ‘1’.")
+		/// Purge old fsUsers.json file
+		configPath, err := getConfigPath()
+		if err != nil {
+			return fmt.Errorf("Unable to retrieve config path. %v", err)
+		}
+
+		configFile := filepath.Join(configPath, "fsUsers.json")
+		removeAll(configFile)
+		return nil
+	}
+	return fmt.Errorf("Failed to migrate unrecognized config version ‘" + cv1.Version + "’.")
+}
+
+// Version '2' to '3' config migration adds new fields and re-orders
+// previous fields. Simplifies config for future additions.
+func migrateV2ToV3() error {
+	cv2, err := loadConfigV2()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("Unable to load config version ‘2’. %v", err)
+	}
+	if cv2.Version != "2" {
+		return nil
+	}
+	srvConfig := &configV3{}
+	srvConfig.Version = "3"
+	srvConfig.Addr = ":9000"
+	srvConfig.Credential = credential{
+		AccessKeyID:     cv2.Credentials.AccessKeyID,
+		SecretAccessKey: cv2.Credentials.SecretAccessKey,
+	}
+	srvConfig.Region = cv2.Credentials.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature V4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = consoleLogger{
+		Enable: true,
+		Level:  "fatal",
+	}
+	flogger := fileLogger{}
+	flogger.Level = "error"
+	if cv2.FileLogger.Filename != "" {
+		flogger.Enable = true
+		flogger.Filename = cv2.FileLogger.Filename
+	}
+	srvConfig.Logger.File = flogger
+
+	slogger := syslogLoggerV3{}
+	slogger.Level = "debug"
+	if cv2.SyslogLogger.Addr != "" {
+		slogger.Enable = true
+		slogger.Addr = cv2.SyslogLogger.Addr
+	}
+	srvConfig.Logger.Syslog = slogger
+
+	qc, err := quick.New(srvConfig)
+	if err != nil {
+		return fmt.Errorf("Unable to initialize config. %v", err)
+	}
+
+	configFile, err := getConfigFile()
+	if err != nil {
+		return fmt.Errorf("Unable to get config file. %v", err)
+	}
+
+	// Migrate the config.
+	err = qc.Save(configFile)
+	if err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘"+cv2.Version+"’ to ‘"+srvConfig.Version+"’ failed. %v", err)
+	}
+
+	console.Println("Migration from version ‘" + cv2.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+	return nil
+}
+
+// Version '3' to '4' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV3ToV4() error {
+	cv3, err := loadConfigV3()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("Unable to load config version ‘3’. %v", err)
+	}
+	if cv3.Version != "3" {
+		return nil
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &configV4{}
+	srvConfig.Version = "4"
+	srvConfig.Credential = cv3.Credential
+	srvConfig.Region = cv3.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv3.Logger.Console
+	srvConfig.Logger.File = cv3.Logger.File
+	srvConfig.Logger.Syslog = cv3.Logger.Syslog
+
+	qc, err := quick.New(srvConfig)
+	if err != nil {
+		return fmt.Errorf("Unable to initialize the quick config. %v", err)
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return fmt.Errorf("Unable to get config file. %v", err)
+	}
+
+	err = qc.Save(configFile)
+	if err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘"+cv3.Version+"’ to ‘"+srvConfig.Version+"’ failed. %v", err)
+	}
+
+	console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+	return nil
+}
+
+// Version '4' to '5' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV4ToV5() error {
+	cv4, err := loadConfigV4()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("Unable to load config version ‘4’. %v", err)
+	}
+	if cv4.Version != "4" {
+		return nil
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &configV5{}
+	srvConfig.Version = "5"
+	srvConfig.Credential = cv4.Credential
+	srvConfig.Region = cv4.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv4.Logger.Console
+	srvConfig.Logger.File = cv4.Logger.File
+	srvConfig.Logger.Syslog = cv4.Logger.Syslog
+	srvConfig.Logger.AMQP.Enable = false
+	srvConfig.Logger.ElasticSearch.Enable = false
+	srvConfig.Logger.Redis.Enable = false
+
+	qc, err := quick.New(srvConfig)
+	if err != nil {
+		return fmt.Errorf("Unable to initialize the quick config. %v", err)
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return fmt.Errorf("Unable to get config file. %v", err)
+	}
+
+	err = qc.Save(configFile)
+	if err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘"+cv4.Version+"’ to ‘"+srvConfig.Version+"’ failed. %v", err)
+	}
+
+	console.Println("Migration from version ‘" + cv4.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+	return nil
+}
+
+// Version '5' to '6' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV5ToV6() error {
+	cv5, err := loadConfigV5()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("Unable to load config version ‘5’. %v", err)
+	}
+	if cv5.Version != "5" {
+		return nil
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &configV6{}
+	srvConfig.Version = "6"
+	srvConfig.Credential = cv5.Credential
+	srvConfig.Region = cv5.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv5.Logger.Console
+	srvConfig.Logger.File = cv5.Logger.File
+	srvConfig.Logger.Syslog = cv5.Logger.Syslog
+
+	srvConfig.Notify.AMQP = map[string]amqpNotify{
+		"1": {
+			Enable:      cv5.Logger.AMQP.Enable,
+			URL:         cv5.Logger.AMQP.URL,
+			Exchange:    cv5.Logger.AMQP.Exchange,
+			RoutingKey:  cv5.Logger.AMQP.RoutingKey,
+			Mandatory:   cv5.Logger.AMQP.Mandatory,
+			Immediate:   cv5.Logger.AMQP.Immediate,
+			Durable:     cv5.Logger.AMQP.Durable,
+			Internal:    cv5.Logger.AMQP.Internal,
+			NoWait:      cv5.Logger.AMQP.NoWait,
+			AutoDeleted: cv5.Logger.AMQP.AutoDeleted,
+		},
+	}
+	srvConfig.Notify.ElasticSearch = map[string]elasticSearchNotify{
+		"1": {
+			Enable: cv5.Logger.ElasticSearch.Enable,
+			URL:    cv5.Logger.ElasticSearch.URL,
+			Index:  cv5.Logger.ElasticSearch.Index,
+		},
+	}
+	srvConfig.Notify.Redis = map[string]redisNotify{
+		"1": {
+			Enable:   cv5.Logger.Redis.Enable,
+			Addr:     cv5.Logger.Redis.Addr,
+			Password: cv5.Logger.Redis.Password,
+			Key:      cv5.Logger.Redis.Key,
+		},
+	}
+
+	qc, err := quick.New(srvConfig)
+	if err != nil {
+		return fmt.Errorf("Unable to initialize the quick config. %v", err)
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return fmt.Errorf("Unable to get config file. %v", err)
+	}
+
+	err = qc.Save(configFile)
+	if err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘"+cv5.Version+"’ to ‘"+srvConfig.Version+"’ failed. %v", err)
+	}
+
+	console.Println("Migration from version ‘" + cv5.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+	return nil
+}
+
+// Version '6' to '7' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV6ToV7() error {
+	cv6, err := loadConfigV6()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("Unable to load config version ‘6’. %v", err)
+	}
+	if cv6.Version != "6" {
+		return nil
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &serverConfigV7{}
+	srvConfig.Version = "7"
+	srvConfig.Credential = cv6.Credential
+	srvConfig.Region = cv6.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv6.Logger.Console
+	srvConfig.Logger.File = cv6.Logger.File
+	srvConfig.Logger.Syslog = cv6.Logger.Syslog
+	srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+	srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+	srvConfig.Notify.Redis = make(map[string]redisNotify)
+	if len(cv6.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		srvConfig.Notify.AMQP = cv6.Notify.AMQP
+	}
+	if len(cv6.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv6.Notify.ElasticSearch
+	}
+	if len(cv6.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis["1"] = redisNotify{}
+	} else {
+		srvConfig.Notify.Redis = cv6.Notify.Redis
+	}
+
+	qc, err := quick.New(srvConfig)
+	if err != nil {
+		return fmt.Errorf("Unable to initialize the quick config. %v", err)
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return fmt.Errorf("Unable to get config file. %v", err)
+	}
+
+	err = qc.Save(configFile)
+	if err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘"+cv6.Version+"’ to ‘"+srvConfig.Version+"’ failed. %v", err)
+	}
+
+	console.Println("Migration from version ‘" + cv6.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+	return nil
+}
+
+// Version '7' to '8' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV7ToV8() error {
+	cv7, err := loadConfigV7()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("Unable to load config version ‘7’. %v", err)
+	}
+	if cv7.Version != "7" {
+		return nil
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &serverConfigV8{}
+	srvConfig.Version = "8"
+	srvConfig.Credential = cv7.Credential
+	srvConfig.Region = cv7.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv7.Logger.Console
+	srvConfig.Logger.File = cv7.Logger.File
+	srvConfig.Logger.Syslog = cv7.Logger.Syslog
+	srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+	srvConfig.Notify.NATS = make(map[string]natsNotify)
+	srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+	srvConfig.Notify.Redis = make(map[string]redisNotify)
+	srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+	if len(cv7.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		srvConfig.Notify.AMQP = cv7.Notify.AMQP
+	}
+	if len(cv7.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS["1"] = natsNotify{}
+	} else {
+		srvConfig.Notify.NATS = cv7.Notify.NATS
+	}
+	if len(cv7.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv7.Notify.ElasticSearch
+	}
+	if len(cv7.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis["1"] = redisNotify{}
+	} else {
+		srvConfig.Notify.Redis = cv7.Notify.Redis
+	}
+
+	qc, err := quick.New(srvConfig)
+	if err != nil {
+		return fmt.Errorf("Unable to initialize the quick config. %v", err)
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return fmt.Errorf("Unable to get config file. %v", err)
+	}
+
+	err = qc.Save(configFile)
+	if err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘"+cv7.Version+"’ to ‘"+srvConfig.Version+"’ failed. %v", err)
+	}
+
+	console.Println("Migration from version ‘" + cv7.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+	return nil
+}
+
+// Version '8' to '9' migration. Adds postgresql notifier
+// configuration, but it's otherwise the same as V8.
+func migrateV8ToV9() error {
+	cv8, err := loadConfigV8()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("Unable to load config version ‘8’. %v", err)
+	}
+	if cv8.Version != "8" {
+		return nil
+	}
+
+	// Copy over fields from V8 into V9 config struct
+	srvConfig := &serverConfigV9{}
+	srvConfig.Version = "9"
+	srvConfig.Credential = cv8.Credential
+	srvConfig.Region = cv8.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv8.Logger.Console
+	srvConfig.Logger.Console.Level = "error"
+	srvConfig.Logger.File = cv8.Logger.File
+	srvConfig.Logger.Syslog = cv8.Logger.Syslog
+
+	// check and set notifiers config
+	if len(cv8.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		srvConfig.Notify.AMQP = cv8.Notify.AMQP
+	}
+	if len(cv8.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]natsNotify)
+		srvConfig.Notify.NATS["1"] = natsNotify{}
+	} else {
+		srvConfig.Notify.NATS = cv8.Notify.NATS
+	}
+	if len(cv8.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv8.Notify.ElasticSearch
+	}
+	if len(cv8.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]redisNotify)
+		srvConfig.Notify.Redis["1"] = redisNotify{}
+	} else {
+		srvConfig.Notify.Redis = cv8.Notify.Redis
+	}
+	if len(cv8.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv8.Notify.PostgreSQL
+	}
+
+	qc, err := quick.New(srvConfig)
+	if err != nil {
+		return fmt.Errorf("Unable to initialize the quick config. %v",
+			err)
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return fmt.Errorf("Unable to get config file. %v", err)
+	}
+
+	err = qc.Save(configFile)
+	if err != nil {
+		return fmt.Errorf(
+			"Failed to migrate config from ‘"+
+				cv8.Version+"’ to ‘"+srvConfig.Version+
+				"’ failed. %v", err,
+		)
+	}
+
+	console.Println(
+		"Migration from version ‘" +
+			cv8.Version + "’ to ‘" + srvConfig.Version +
+			"’ completed successfully.",
+	)
+	return nil
+}
+
+// Version '9' to '10' migration. Remove syslog config
+// but it's otherwise the same as V9.
+func migrateV9ToV10() error {
+	cv9, err := loadConfigV9()
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("Unable to load config version ‘9’. %v", err)
+	}
+	if cv9.Version != "9" {
+		return nil
+	}
+
+	// Copy over fields from V9 into V10 config struct
+	srvConfig := &serverConfigV10{}
+	srvConfig.Version = "10"
+	srvConfig.Credential = cv9.Credential
+	srvConfig.Region = cv9.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv9.Logger.Console
+	srvConfig.Logger.File = cv9.Logger.File
+
+	// check and set notifiers config
+	if len(cv9.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		srvConfig.Notify.AMQP = cv9.Notify.AMQP
+	}
+	if len(cv9.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]natsNotify)
+		srvConfig.Notify.NATS["1"] = natsNotify{}
+	} else {
+		srvConfig.Notify.NATS = cv9.Notify.NATS
+	}
+	if len(cv9.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv9.Notify.ElasticSearch
+	}
+	if len(cv9.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]redisNotify)
+		srvConfig.Notify.Redis["1"] = redisNotify{}
+	} else {
+		srvConfig.Notify.Redis = cv9.Notify.Redis
+	}
+	if len(cv9.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv9.Notify.PostgreSQL
+	}
+
+	qc, err := quick.New(srvConfig)
+	if err != nil {
+		return fmt.Errorf("Unable to initialize the quick config. %v",
+			err)
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return fmt.Errorf("Unable to get config file. %v", err)
+	}
+
+	err = qc.Save(configFile)
+	if err != nil {
+		return fmt.Errorf(
+			"Failed to migrate config from ‘"+
+				cv9.Version+"’ to ‘"+srvConfig.Version+
+				"’ failed. %v", err,
+		)
+	}
+
+	console.Println(
+		"Migration from version ‘" +
+			cv9.Version + "’ to ‘" + srvConfig.Version +
+			"’ completed successfully.",
+	)
+	return nil
+}

cmd/config-migrate_test.go

@@ -0,0 +1,203 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+// Test if config v1 is purged
+func TestServerConfigMigrateV1(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	setGlobalConfigPath(rootPath)
+
+	// Create a V1 config json file and store it
+	configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
+	configPath := rootPath + "/fsUsers.json"
+	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0644); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	// Fire a migrateConfig()
+	if err := migrateConfig(); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+	// Check if config v1 is removed from filesystem
+	if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
+		t.Fatal("Config V1 file is not purged")
+	}
+
+	// Initialize server config and check again if everything is fine
+	if _, err := initConfig(); err != nil {
+		t.Fatalf("Unable to initialize from updated config file %s", err)
+	}
+}
+
+// Test if all migrate code returns nil when config file does not
+// exist
+func TestServerConfigMigrateInexistentConfig(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	setGlobalConfigPath(rootPath)
+	configPath := rootPath + "/" + globalMinioConfigFile
+
+	// Remove config file
+	if err := os.Remove(configPath); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	if err := migrateV2ToV3(); err != nil {
+		t.Fatal("migrate v2 to v3 should succeed when no config file is found")
+	}
+	if err := migrateV3ToV4(); err != nil {
+		t.Fatal("migrate v3 to v4 should succeed when no config file is found")
+	}
+	if err := migrateV4ToV5(); err != nil {
+		t.Fatal("migrate v4 to v5 should succeed when no config file is found")
+	}
+	if err := migrateV5ToV6(); err != nil {
+		t.Fatal("migrate v5 to v6 should succeed when no config file is found")
+	}
+	if err := migrateV6ToV7(); err != nil {
+		t.Fatal("migrate v6 to v7 should succeed when no config file is found")
+	}
+	if err := migrateV7ToV8(); err != nil {
+		t.Fatal("migrate v7 to v8 should succeed when no config file is found")
+	}
+	if err := migrateV8ToV9(); err != nil {
+		t.Fatal("migrate v8 to v9 should succeed when no config file is found")
+	}
+	if err := migrateV9ToV10(); err != nil {
+		t.Fatal("migrate v9 to v10 should succeed when no config file is found")
+	}
+}
+
+// Test if a config migration from v2 to v10 is successfully done
+func TestServerConfigMigrateV2toV10(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	setGlobalConfigPath(rootPath)
+	configPath := rootPath + "/" + globalMinioConfigFile
+
+	// Create a corrupted config file
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0644); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+	// Fire a migrateConfig()
+	if err := migrateConfig(); err == nil {
+		t.Fatal("migration should fail with corrupted config file")
+	}
+
+	accessKey := "accessfoo"
+	secretKey := "secretfoo"
+
+	// Create a V2 config json file and store it
+	configJSON := "{ \"version\":\"2\", \"credentials\": {\"accessKeyId\":\"" + accessKey + "\", \"secretAccessKey\":\"" + secretKey + "\", \"region\":\"us-east-1\"}, \"mongoLogger\":{\"addr\":\"127.0.0.1:3543\", \"db\":\"foodb\", \"collection\":\"foo\"}, \"syslogLogger\":{\"network\":\"127.0.0.1:543\", \"addr\":\"addr\"}, \"fileLogger\":{\"filename\":\"log.out\"}}"
+	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0644); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+	// Fire a migrateConfig()
+	if err := migrateConfig(); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	// Initialize server config and check again if everything is fine
+	if _, err := initConfig(); err != nil {
+		t.Fatalf("Unable to initialize from updated config file %s", err)
+	}
+
+	// Check the version number in the upgraded config file
+	expectedVersion := globalMinioConfigVersion
+	if serverConfig.Version != expectedVersion {
+		t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
+	}
+
+	// Check if accessKey and secretKey are not altered during migration
+	if serverConfig.Credential.AccessKeyID != accessKey {
+		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, serverConfig.Credential.AccessKeyID)
+	}
+	if serverConfig.Credential.SecretAccessKey != secretKey {
+		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, serverConfig.Credential.SecretAccessKey)
+	}
+
+	// Initialize server config and check again if everything is fine
+	if _, err := initConfig(); err != nil {
+		t.Fatalf("Unable to initialize from updated config file %s", err)
+	}
+}
+
+// Test if all migrate code returns error with corrupted config files
+func TestServerConfigMigrateFaultyConfig(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	setGlobalConfigPath(rootPath)
+	configPath := rootPath + "/" + globalMinioConfigFile
+
+	// Create a corrupted config file
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\""), 0644); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	// Test different migrate versions and be sure they are returning an error
+	if err := migrateV2ToV3(); err == nil {
+		t.Fatal("migrateConfigV2ToV3() should fail with a corrupted json")
+	}
+	if err := migrateV3ToV4(); err == nil {
+		t.Fatal("migrateConfigV3ToV4() should fail with a corrupted json")
+	}
+	if err := migrateV4ToV5(); err == nil {
+		t.Fatal("migrateConfigV4ToV5() should fail with a corrupted json")
+	}
+	if err := migrateV5ToV6(); err == nil {
+		t.Fatal("migrateConfigV5ToV6() should fail with a corrupted json")
+	}
+	if err := migrateV6ToV7(); err == nil {
+		t.Fatal("migrateConfigV6ToV7() should fail with a corrupted json")
+	}
+	if err := migrateV7ToV8(); err == nil {
+		t.Fatal("migrateConfigV7ToV8() should fail with a corrupted json")
+	}
+	if err := migrateV8ToV9(); err == nil {
+		t.Fatal("migrateConfigV8ToV9() should fail with a corrupted json")
+	}
+	if err := migrateV9ToV10(); err == nil {
+		t.Fatal("migrateConfigV9ToV10() should fail with a corrupted json")
+	}
+}

cmd/config-old.go

@@ -0,0 +1,445 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/minio/minio/pkg/quick"
+)
+
+/////////////////// Config V1 ///////////////////
+type configV1 struct {
+	Version         string `json:"version"`
+	AccessKeyID     string `json:"accessKeyId"`
+	SecretAccessKey string `json:"secretAccessKey"`
+}
+
+// loadConfigV1 load config
+func loadConfigV1() (*configV1, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return nil, err
+	}
+	configFile := filepath.Join(configPath, "fsUsers.json")
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV1{}
+	c.Version = "1"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+/////////////////// Config V2 ///////////////////
+type configV2 struct {
+	Version     string `json:"version"`
+	Credentials struct {
+		AccessKeyID     string `json:"accessKeyId"`
+		SecretAccessKey string `json:"secretAccessKey"`
+		Region          string `json:"region"`
+	} `json:"credentials"`
+	MongoLogger struct {
+		Addr       string `json:"addr"`
+		DB         string `json:"db"`
+		Collection string `json:"collection"`
+	} `json:"mongoLogger"`
+	SyslogLogger struct {
+		Network string `json:"network"`
+		Addr    string `json:"addr"`
+	} `json:"syslogLogger"`
+	FileLogger struct {
+		Filename string `json:"filename"`
+	} `json:"fileLogger"`
+}
+
+// loadConfigV2 load config version '2'.
+func loadConfigV2() (*configV2, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV2{}
+	c.Version = "2"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+/////////////////// Config V3 ///////////////////
+
+// backendV3 type.
+type backendV3 struct {
+	Type  string   `json:"type"`
+	Disk  string   `json:"disk,omitempty"`
+	Disks []string `json:"disks,omitempty"`
+}
+
+// syslogLogger v3
+type syslogLoggerV3 struct {
+	Enable bool   `json:"enable"`
+	Addr   string `json:"address"`
+	Level  string `json:"level"`
+}
+
+// loggerV3 type.
+type loggerV3 struct {
+	Console struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+	}
+	File struct {
+		Enable   bool   `json:"enable"`
+		Filename string `json:"fileName"`
+		Level    string `json:"level"`
+	}
+	Syslog struct {
+		Enable bool   `json:"enable"`
+		Addr   string `json:"address"`
+		Level  string `json:"level"`
+	} `json:"syslog"`
+	// Add new loggers here.
+}
+
+// configV3 server configuration version '3'.
+type configV3 struct {
+	Version string `json:"version"`
+
+	// Backend configuration.
+	Backend backendV3 `json:"backend"`
+
+	// http Server configuration.
+	Addr string `json:"address"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV3 `json:"logger"`
+}
+
+// loadConfigV3 load config version '3'.
+func loadConfigV3() (*configV3, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV3{}
+	c.Version = "3"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// logger type representing version '4' logger config.
+type loggerV4 struct {
+	Console struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+	} `json:"console"`
+	File struct {
+		Enable   bool   `json:"enable"`
+		Filename string `json:"fileName"`
+		Level    string `json:"level"`
+	} `json:"file"`
+	Syslog struct {
+		Enable bool   `json:"enable"`
+		Addr   string `json:"address"`
+		Level  string `json:"level"`
+	} `json:"syslog"`
+}
+
+// configV4 server configuration version '4'.
+type configV4 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV4 `json:"logger"`
+}
+
+// loadConfigV4 load config version '4'.
+func loadConfigV4() (*configV4, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV4{}
+	c.Version = "4"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// logger type representing version '5' logger config.
+type loggerV5 struct {
+	Console struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+	} `json:"console"`
+	File struct {
+		Enable   bool   `json:"enable"`
+		Filename string `json:"fileName"`
+		Level    string `json:"level"`
+	} `json:"file"`
+	Syslog struct {
+		Enable bool   `json:"enable"`
+		Addr   string `json:"address"`
+		Level  string `json:"level"`
+	} `json:"syslog"`
+	AMQP struct {
+		Enable       bool   `json:"enable"`
+		Level        string `json:"level"`
+		URL          string `json:"url"`
+		Exchange     string `json:"exchange"`
+		RoutingKey   string `json:"routingKey"`
+		ExchangeType string `json:"exchangeType"`
+		Mandatory    bool   `json:"mandatory"`
+		Immediate    bool   `json:"immediate"`
+		Durable      bool   `json:"durable"`
+		Internal     bool   `json:"internal"`
+		NoWait       bool   `json:"noWait"`
+		AutoDeleted  bool   `json:"autoDeleted"`
+	} `json:"amqp"`
+	ElasticSearch struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+		URL    string `json:"url"`
+		Index  string `json:"index"`
+	} `json:"elasticsearch"`
+	Redis struct {
+		Enable   bool   `json:"enable"`
+		Level    string `json:"level"`
+		Addr     string `json:"address"`
+		Password string `json:"password"`
+		Key      string `json:"key"`
+	} `json:"redis"`
+}
+
+// configV5 server configuration version '5'.
+type configV5 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV5 `json:"logger"`
+}
+
+// loadConfigV5 load config version '5'.
+func loadConfigV5() (*configV5, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV5{}
+	c.Version = "5"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+type loggerV6 struct {
+	Console consoleLogger  `json:"console"`
+	File    fileLogger     `json:"file"`
+	Syslog  syslogLoggerV3 `json:"syslog"`
+}
+
+// configV6 server configuration version '6'.
+type configV6 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV6 `json:"logger"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// loadConfigV6 load config version '6'.
+func loadConfigV6() (*configV6, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV6{}
+	c.Version = "6"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// configV7 server configuration version '7'.
+type serverConfigV7 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV6 `json:"logger"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Read Write mutex.
+	rwMutex *sync.RWMutex
+}
+
+// loadConfigV7 load config version '7'.
+func loadConfigV7() (*serverConfigV7, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &serverConfigV7{}
+	c.Version = "7"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// serverConfigV8 server configuration version '8'. Adds NATS notifier
+// configuration.
+type serverConfigV8 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV6 `json:"logger"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Read Write mutex.
+	rwMutex *sync.RWMutex
+}
+
+// loadConfigV8 load config version '8'.
+func loadConfigV8() (*serverConfigV8, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &serverConfigV8{}
+	c.Version = "8"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// serverConfigV9 server configuration version '9'. Adds PostgreSQL
+// notifier configuration.
+type serverConfigV9 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV6 `json:"logger"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Read Write mutex.
+	rwMutex *sync.RWMutex
+}
+
+func loadConfigV9() (*serverConfigV9, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	srvCfg := &serverConfigV9{}
+	srvCfg.Version = "9"
+	srvCfg.rwMutex = &sync.RWMutex{}
+	qc, err := quick.New(srvCfg)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return srvCfg, nil
+}

cmd/config-v10.go

@@ -0,0 +1,320 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"sync"
+
+	"github.com/minio/minio/pkg/quick"
+)
+
+// Read Write mutex for safe access to ServerConfig.
+var serverConfigMu sync.RWMutex
+
+// serverConfigV10 server configuration version '10' which is like version '9'
+// except it drops support of syslog config.
+type serverConfigV10 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger logger `json:"logger"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// initConfig - initialize server config and indicate if we are creating a new file or we are just loading
+func initConfig() (bool, error) {
+	if !isConfigFileExists() {
+		// Initialize server config.
+		srvCfg := &serverConfigV10{}
+		srvCfg.Version = globalMinioConfigVersion
+		srvCfg.Region = "us-east-1"
+		srvCfg.Credential = mustGenAccessKeys()
+
+		// Enable console logger by default on a fresh run.
+		srvCfg.Logger.Console = consoleLogger{
+			Enable: true,
+			Level:  "error",
+		}
+
+		// Make sure to initialize notification configs.
+		srvCfg.Notify.AMQP = make(map[string]amqpNotify)
+		srvCfg.Notify.AMQP["1"] = amqpNotify{}
+		srvCfg.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvCfg.Notify.ElasticSearch["1"] = elasticSearchNotify{}
+		srvCfg.Notify.Redis = make(map[string]redisNotify)
+		srvCfg.Notify.Redis["1"] = redisNotify{}
+		srvCfg.Notify.NATS = make(map[string]natsNotify)
+		srvCfg.Notify.NATS["1"] = natsNotify{}
+		srvCfg.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
+		srvCfg.Notify.PostgreSQL["1"] = postgreSQLNotify{}
+
+		// Create config path.
+		err := createConfigPath()
+		if err != nil {
+			return false, err
+		}
+		// hold the mutex lock before a new config is assigned.
+		// Save the new config globally.
+		// unlock the mutex.
+		serverConfigMu.Lock()
+		serverConfig = srvCfg
+		serverConfigMu.Unlock()
+
+		// Save config into file.
+		return true, serverConfig.Save()
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return false, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return false, err
+	}
+	srvCfg := &serverConfigV10{}
+	srvCfg.Version = globalMinioConfigVersion
+	qc, err := quick.New(srvCfg)
+	if err != nil {
+		return false, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return false, err
+	}
+
+	// hold the mutex lock before a new config is assigned.
+	serverConfigMu.Lock()
+	// Save the loaded config globally.
+	serverConfig = srvCfg
+	serverConfigMu.Unlock()
+	// Set the version properly after the unmarshalled json is loaded.
+	serverConfig.Version = globalMinioConfigVersion
+
+	return false, nil
+}
+
+// serverConfig server config.
+var serverConfig *serverConfigV10
+
+// GetVersion get current config version.
+func (s serverConfigV10) GetVersion() string {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Version
+}
+
+/// Logger related.
+
+func (s *serverConfigV10) SetAMQPNotifyByID(accountID string, amqpn amqpNotify) {
+	serverConfigMu.Lock()
+	defer serverConfigMu.Unlock()
+
+	s.Notify.AMQP[accountID] = amqpn
+}
+
+func (s serverConfigV10) GetAMQP() map[string]amqpNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Notify.AMQP
+}
+
+// GetAMQPNotify get current AMQP logger.
+func (s serverConfigV10) GetAMQPNotifyByID(accountID string) amqpNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Notify.AMQP[accountID]
+}
+
+//
+func (s *serverConfigV10) SetNATSNotifyByID(accountID string, natsn natsNotify) {
+	serverConfigMu.Lock()
+	defer serverConfigMu.Unlock()
+
+	s.Notify.NATS[accountID] = natsn
+}
+
+func (s serverConfigV10) GetNATS() map[string]natsNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+	return s.Notify.NATS
+}
+
+// GetNATSNotify get current NATS logger.
+func (s serverConfigV10) GetNATSNotifyByID(accountID string) natsNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Notify.NATS[accountID]
+}
+
+func (s *serverConfigV10) SetElasticSearchNotifyByID(accountID string, esNotify elasticSearchNotify) {
+	serverConfigMu.Lock()
+	defer serverConfigMu.Unlock()
+
+	s.Notify.ElasticSearch[accountID] = esNotify
+}
+
+func (s serverConfigV10) GetElasticSearch() map[string]elasticSearchNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Notify.ElasticSearch
+}
+
+// GetElasticSearchNotify get current ElasicSearch logger.
+func (s serverConfigV10) GetElasticSearchNotifyByID(accountID string) elasticSearchNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Notify.ElasticSearch[accountID]
+}
+
+func (s *serverConfigV10) SetRedisNotifyByID(accountID string, rNotify redisNotify) {
+	serverConfigMu.Lock()
+	defer serverConfigMu.Unlock()
+
+	s.Notify.Redis[accountID] = rNotify
+}
+
+func (s serverConfigV10) GetRedis() map[string]redisNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Notify.Redis
+}
+
+// GetRedisNotify get current Redis logger.
+func (s serverConfigV10) GetRedisNotifyByID(accountID string) redisNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Notify.Redis[accountID]
+}
+
+func (s *serverConfigV10) SetPostgreSQLNotifyByID(accountID string, pgn postgreSQLNotify) {
+	serverConfigMu.Lock()
+	defer serverConfigMu.Unlock()
+
+	s.Notify.PostgreSQL[accountID] = pgn
+}
+
+func (s serverConfigV10) GetPostgreSQL() map[string]postgreSQLNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Notify.PostgreSQL
+}
+
+func (s serverConfigV10) GetPostgreSQLNotifyByID(accountID string) postgreSQLNotify {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Notify.PostgreSQL[accountID]
+}
+
+// SetFileLogger set new file logger.
+func (s *serverConfigV10) SetFileLogger(flogger fileLogger) {
+	serverConfigMu.Lock()
+	defer serverConfigMu.Unlock()
+
+	s.Logger.File = flogger
+}
+
+// GetFileLogger get current file logger.
+func (s serverConfigV10) GetFileLogger() fileLogger {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Logger.File
+}
+
+// SetConsoleLogger set new console logger.
+func (s *serverConfigV10) SetConsoleLogger(clogger consoleLogger) {
+	serverConfigMu.Lock()
+	defer serverConfigMu.Unlock()
+
+	s.Logger.Console = clogger
+}
+
+// GetConsoleLogger get current console logger.
+func (s serverConfigV10) GetConsoleLogger() consoleLogger {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Logger.Console
+}
+
+// SetRegion set new region.
+func (s *serverConfigV10) SetRegion(region string) {
+	serverConfigMu.Lock()
+	defer serverConfigMu.Unlock()
+
+	s.Region = region
+}
+
+// GetRegion get current region.
+func (s serverConfigV10) GetRegion() string {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Region
+}
+
+// SetCredentials set new credentials.
+func (s *serverConfigV10) SetCredential(creds credential) {
+	serverConfigMu.Lock()
+	defer serverConfigMu.Unlock()
+
+	s.Credential = creds
+}
+
+// GetCredentials get current credentials.
+func (s serverConfigV10) GetCredential() credential {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	return s.Credential
+}
+
+// Save config.
+func (s serverConfigV10) Save() error {
+	serverConfigMu.RLock()
+	defer serverConfigMu.RUnlock()
+
+	// get config file.
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err
+	}
+
+	// initialize quick.
+	qc, err := quick.New(&s)
+	if err != nil {
+		return err
+	}
+
+	// Save config file.
+	return qc.Save(configFile)
+}

cmd/config-v10_test.go

@@ -0,0 +1,98 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestServerConfig(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	if serverConfig.GetRegion() != "us-east-1" {
+		t.Errorf("Expecting region `us-east-1` found %s", serverConfig.GetRegion())
+	}
+
+	// Set new region and verify.
+	serverConfig.SetRegion("us-west-1")
+	if serverConfig.GetRegion() != "us-west-1" {
+		t.Errorf("Expecting region `us-west-1` found %s", serverConfig.GetRegion())
+	}
+
+	// Set new amqp notification id.
+	serverConfig.SetAMQPNotifyByID("2", amqpNotify{})
+	savedNotifyCfg1 := serverConfig.GetAMQPNotifyByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg1, amqpNotify{}) {
+		t.Errorf("Expecting AMQP config %#v found %#v", amqpNotify{}, savedNotifyCfg1)
+	}
+
+	// Set new elastic search notification id.
+	serverConfig.SetElasticSearchNotifyByID("2", elasticSearchNotify{})
+	savedNotifyCfg2 := serverConfig.GetElasticSearchNotifyByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg2, elasticSearchNotify{}) {
+		t.Errorf("Expecting Elasticsearch config %#v found %#v", elasticSearchNotify{}, savedNotifyCfg2)
+	}
+
+	// Set new redis notification id.
+	serverConfig.SetRedisNotifyByID("2", redisNotify{})
+	savedNotifyCfg3 := serverConfig.GetRedisNotifyByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg3, redisNotify{}) {
+		t.Errorf("Expecting Redis config %#v found %#v", redisNotify{}, savedNotifyCfg3)
+	}
+
+	// Set new console logger.
+	serverConfig.SetConsoleLogger(consoleLogger{
+		Enable: true,
+	})
+	consoleCfg := serverConfig.GetConsoleLogger()
+	if !reflect.DeepEqual(consoleCfg, consoleLogger{Enable: true}) {
+		t.Errorf("Expecting console logger config %#v found %#v", consoleLogger{Enable: true}, consoleCfg)
+	}
+
+	// Set new file logger.
+	serverConfig.SetFileLogger(fileLogger{
+		Enable: true,
+	})
+	fileCfg := serverConfig.GetFileLogger()
+	if !reflect.DeepEqual(fileCfg, fileLogger{Enable: true}) {
+		t.Errorf("Expecting file logger config %#v found %#v", fileLogger{Enable: true}, consoleCfg)
+	}
+
+	// Match version.
+	if serverConfig.GetVersion() != globalMinioConfigVersion {
+		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), globalMinioConfigVersion)
+	}
+
+	// Attempt to save.
+	if err := serverConfig.Save(); err != nil {
+		t.Fatalf("Unable to save updated config file %s", err)
+	}
+
+	// Do this only once here.
+	setGlobalConfigPath(rootPath)
+
+	// Initialize server config.
+	if _, err := initConfig(); err != nil {
+		t.Fatalf("Unable to initialize from updated config file %s", err)
+	}
+}

cmd/config.go

@@ -0,0 +1,93 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/minio/go-homedir"
+)
+
+// configPath for custom config path only for testing purposes
+var customConfigPath string
+var configMu sync.Mutex
+
+// Sets a new config path.
+func setGlobalConfigPath(configPath string) {
+	configMu.Lock()
+	defer configMu.Unlock()
+	customConfigPath = configPath
+}
+
+// getConfigPath get server config path
+func getConfigPath() (string, error) {
+	configMu.Lock()
+	defer configMu.Unlock()
+
+	if customConfigPath != "" {
+		return customConfigPath, nil
+	}
+	homeDir, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+	configPath := filepath.Join(homeDir, globalMinioConfigDir)
+	return configPath, nil
+}
+
+// mustGetConfigPath must get server config path.
+func mustGetConfigPath() string {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return ""
+	}
+	return configPath
+}
+
+// createConfigPath create server config path.
+func createConfigPath() error {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return err
+	}
+	return os.MkdirAll(configPath, 0700)
+}
+
+// isConfigFileExists - returns true if config file exists.
+func isConfigFileExists() bool {
+	path, err := getConfigFile()
+	if err != nil {
+		return false
+	}
+	st, err := os.Stat(path)
+	// If file exists and is regular return true.
+	if err == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// getConfigFile get server config file.
+func getConfigFile() (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(configPath, globalMinioConfigFile), nil
+}

cmd/crossdomain-xml-handler.go

@@ -0,0 +1,49 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "net/http"
+
+// Standard cross domain policy information located at https://s3.amazonaws.com/crossdomain.xml
+const crossDomainXML = `<?xml version="1.0"?><!DOCTYPE cross-domain-policy SYSTEM "http://www.adobe.com/xml/dtds/cross-domain-policy.dtd"><cross-domain-policy><allow-access-from domain="*" secure="false" /></cross-domain-policy>`
+
+// Cross domain policy implements http.Handler interface, implementing a custom ServerHTTP.
+type crossDomainPolicy struct {
+	handler http.Handler
+}
+
+// A cross-domain policy file is an XML document that grants a web client, such as Adobe Flash Player
+// or Adobe Acrobat (though not necessarily limited to these), permission to handle data across domains.
+// When clients request content hosted on a particular source domain and that content make requests
+// directed towards a domain other than its own, the remote domain needs to host a cross-domain
+// policy file that grants access to the source domain, allowing the client to continue the transaction.
+func setCrossDomainPolicy(h http.Handler) http.Handler {
+	return crossDomainPolicy{handler: h}
+}
+
+func (c crossDomainPolicy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Look for 'crossdomain.xml' in the incoming request.
+	switch r.URL.Path {
+	case "/crossdomain.xml":
+		// Write the standard cross domain policy xml.
+		w.Write([]byte(crossDomainXML))
+		// Request completed, no need to serve to other handlers.
+		return
+	}
+	// Continue to serve the request further.
+	c.handler.ServeHTTP(w, r)
+}

cmd/damerau-levenshtein.go

@@ -0,0 +1,66 @@
+/*
+ * Minio Client (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"math"
+)
+
+// Returns the minimum value of a slice of integers
+func minimum(integers []int) (minVal int) {
+	minVal = math.MaxInt32
+	for _, v := range integers {
+		if v < minVal {
+			minVal = v
+		}
+	}
+	return
+}
+
+// DamerauLevenshteinDistance calculates distance between two strings using an algorithm
+// described in https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance
+func DamerauLevenshteinDistance(a string, b string) int {
+	d := make([][]int, len(a)+1)
+	for i := 1; i <= len(a)+1; i++ {
+		d[i-1] = make([]int, len(b)+1)
+	}
+	for i := 0; i <= len(a); i++ {
+		d[i][0] = i
+	}
+	for j := 0; j <= len(b); j++ {
+		d[0][j] = j
+	}
+	var cost int
+	for i := 1; i <= len(a); i++ {
+		for j := 1; j <= len(b); j++ {
+			if a[i-1] == b[j-1] {
+				cost = 0
+			} else {
+				cost = 1
+			}
+			d[i][j] = minimum([]int{
+				d[i-1][j] + 1,
+				d[i][j-1] + 1,
+				d[i-1][j-1] + cost,
+			})
+			if i > 1 && j > 1 && a[i-1] == b[j-2] && a[i-2] == b[j-1] {
+				d[i][j] = minimum([]int{d[i][j], d[i-2][j-2] + cost}) // transposition
+			}
+		}
+	}
+	return d[len(a)][len(b)]
+}

cmd/damerau-levenshtein_test.go

@@ -0,0 +1,66 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"math"
+	"testing"
+)
+
+// Test minimum function which calculates the minimal value in a list of integers
+func TestMinimum(t *testing.T) {
+	type testCase struct {
+		listval  []int
+		expected int
+		pass     bool
+	}
+	testCases := []testCase{
+		{listval: []int{3, 4, 15}, expected: 3},
+		{listval: []int{}, expected: math.MaxInt32},
+	}
+	// Validate all the test cases.
+	for i, tt := range testCases {
+		val := minimum(tt.listval)
+		if val != tt.expected {
+			t.Errorf("Test %d:, Expected %d, got %d", i+1, tt.expected, val)
+		}
+	}
+}
+
+// Test DamerauLevenshtein which calculates the difference distance between two words
+func TestDamerauLevenshtein(t *testing.T) {
+	type testCase struct {
+		word1    string
+		word2    string
+		distance int
+	}
+	testCases := []testCase{
+		{word1: "", word2: "", distance: 0},
+		{word1: "a", word2: "a", distance: 0},
+		{word1: "a", word2: "b", distance: 1},
+		{word1: "rm", word2: "tm", distance: 1},
+		{word1: "version", word2: "evrsion", distance: 1},
+		{word1: "version", word2: "bersio", distance: 2},
+	}
+	// Validate all the test cases.
+	for i, tt := range testCases {
+		d := DamerauLevenshteinDistance(tt.word1, tt.word2)
+		if d != tt.distance {
+			t.Errorf("Test %d:, Expected %d, got %d", i+1, tt.distance, d)
+		}
+	}
+}

cmd/erasure-createfile.go

@@ -0,0 +1,145 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/hex"
+	"hash"
+	"io"
+	"sync"
+
+	"github.com/klauspost/reedsolomon"
+)
+
+// erasureCreateFile - writes an entire stream by erasure coding to
+// all the disks, writes also calculate individual block's checksum
+// for future bit-rot protection.
+func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, blockSize int64, dataBlocks int, parityBlocks int, algo string, writeQuorum int) (bytesWritten int64, checkSums []string, err error) {
+	// Allocated blockSized buffer for reading from incoming stream.
+	buf := make([]byte, blockSize)
+
+	hashWriters := newHashWriters(len(disks), algo)
+
+	// Read until io.EOF, erasure codes data and writes to all disks.
+	for {
+		var blocks [][]byte
+		n, rErr := io.ReadFull(reader, buf)
+		// FIXME: this is a bug in Golang, n == 0 and err ==
+		// io.ErrUnexpectedEOF for io.ReadFull function.
+		if n == 0 && rErr == io.ErrUnexpectedEOF {
+			return 0, nil, traceError(rErr)
+		}
+		if rErr == io.EOF {
+			// We have reached EOF on the first byte read, io.Reader
+			// must be 0bytes, we don't need to erasure code
+			// data. Will create a 0byte file instead.
+			if bytesWritten == 0 {
+				blocks = make([][]byte, len(disks))
+				rErr = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum)
+				if rErr != nil {
+					return 0, nil, rErr
+				}
+			} // else we have reached EOF after few reads, no need to
+			// add an additional 0bytes at the end.
+			break
+		}
+		if rErr != nil && rErr != io.ErrUnexpectedEOF {
+			return 0, nil, traceError(rErr)
+		}
+		if n > 0 {
+			// Returns encoded blocks.
+			var enErr error
+			blocks, enErr = encodeData(buf[0:n], dataBlocks, parityBlocks)
+			if enErr != nil {
+				return 0, nil, enErr
+			}
+
+			// Write to all disks.
+			if err = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum); err != nil {
+				return 0, nil, err
+			}
+			bytesWritten += int64(n)
+		}
+	}
+
+	checkSums = make([]string, len(disks))
+	for i := range checkSums {
+		checkSums[i] = hex.EncodeToString(hashWriters[i].Sum(nil))
+	}
+	return bytesWritten, checkSums, nil
+}
+
+// encodeData - encodes incoming data buffer into
+// dataBlocks+parityBlocks returns a 2 dimensional byte array.
+func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) {
+	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
+	if err != nil {
+		return nil, traceError(err)
+	}
+	// Split the input buffer into data and parity blocks.
+	var blocks [][]byte
+	blocks, err = rs.Split(dataBuffer)
+	if err != nil {
+		return nil, traceError(err)
+	}
+
+	// Encode parity blocks using data blocks.
+	err = rs.Encode(blocks)
+	if err != nil {
+		return nil, traceError(err)
+	}
+
+	// Return encoded blocks.
+	return blocks, nil
+}
+
+// appendFile - append data buffer at path.
+func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hashWriters []hash.Hash, writeQuorum int) (err error) {
+	var wg = &sync.WaitGroup{}
+	var wErrs = make([]error, len(disks))
+	// Write encoded data to quorum disks in parallel.
+	for index, disk := range disks {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		// Write encoded data in routine.
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			wErr := disk.AppendFile(volume, path, enBlocks[index])
+			if wErr != nil {
+				wErrs[index] = traceError(wErr)
+				return
+			}
+
+			// Calculate hash for each blocks.
+			hashWriters[index].Write(enBlocks[index])
+
+			// Successfully wrote.
+			wErrs[index] = nil
+		}(index, disk)
+	}
+
+	// Wait for all the appends to finish.
+	wg.Wait()
+
+	// Do we have write quorum?.
+	if !isDiskQuorum(wErrs, writeQuorum) {
+		return traceError(errXLWriteQuorum)
+	}
+	return reduceWriteQuorumErrs(wErrs, objectOpIgnoredErrs, writeQuorum)
+}

cmd/erasure-createfile_test.go

@@ -0,0 +1,204 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/rand"
+	"testing"
+
+	humanize "github.com/dustin/go-humanize"
+	"github.com/klauspost/reedsolomon"
+)
+
+// Simulates a faulty disk for AppendFile()
+type AppendDiskDown struct {
+	*posix
+}
+
+func (a AppendDiskDown) AppendFile(volume string, path string, buf []byte) error {
+	return errFaultyDisk
+}
+
+// Test erasureCreateFile()
+func TestErasureCreateFile(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 1MiB with random data.
+	data := make([]byte, 1*humanize.MiByte)
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Test when all disks are up.
+	size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	// 2 disks down.
+	disks[4] = AppendDiskDown{disks[4].(*posix)}
+	disks[5] = AppendDiskDown{disks[5].(*posix)}
+
+	// Test when two disks are down.
+	size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	// 4 more disks down. 6 disks down in total.
+	disks[6] = AppendDiskDown{disks[6].(*posix)}
+	disks[7] = AppendDiskDown{disks[7].(*posix)}
+	disks[8] = AppendDiskDown{disks[8].(*posix)}
+	disks[9] = AppendDiskDown{disks[9].(*posix)}
+
+	size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	// 1 more disk down. 7 disk down in total. Should return quorum error.
+	disks[10] = AppendDiskDown{disks[10].(*posix)}
+	_, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if errorCause(err) != errXLWriteQuorum {
+		t.Errorf("erasureCreateFile return value: expected errXLWriteQuorum, got %s", err)
+	}
+}
+
+// TestErasureEncode checks for encoding for different data sets.
+func TestErasureEncode(t *testing.T) {
+	// Collection of cases for encode cases.
+	testEncodeCases := []struct {
+		inputData         []byte
+		inputDataBlocks   int
+		inputParityBlocks int
+
+		shouldPass  bool
+		expectedErr error
+	}{
+		// TestCase - 1.
+		// Regular data encoded.
+		{
+			[]byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."),
+			8,
+			8,
+			true,
+			nil,
+		},
+		// TestCase - 2.
+		// Empty data errors out.
+		{
+			[]byte(""),
+			8,
+			8,
+			false,
+			reedsolomon.ErrShortData,
+		},
+		// TestCase - 3.
+		// Single byte encoded.
+		{
+			[]byte("1"),
+			4,
+			4,
+			true,
+			nil,
+		},
+		// TestCase - 4.
+		// test case with negative data block.
+		{
+			[]byte("1"),
+			-1,
+			8,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 5.
+		// test case with negative parity block.
+		{
+			[]byte("1"),
+			8,
+			-1,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 6.
+		// test case with zero data block.
+		{
+			[]byte("1"),
+			0,
+			8,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 7.
+		// test case with zero parity block.
+		{
+			[]byte("1"),
+			8,
+			0,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 8.
+		// test case with data + parity blocks > 255.
+		// expected to fail with Error Max Shard number.
+		{
+			[]byte("1"),
+			128,
+			128,
+			false,
+			reedsolomon.ErrMaxShardNum,
+		},
+	}
+
+	// Test encode cases.
+	for i, testCase := range testEncodeCases {
+		_, actualErr := encodeData(testCase.inputData, testCase.inputDataBlocks, testCase.inputParityBlocks)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if errorCause(actualErr) != testCase.expectedErr {
+				t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
+			}
+		}
+	}
+}

cmd/erasure-healfile.go

@@ -0,0 +1,84 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "encoding/hex"
+
+// Heals the erasure coded file. reedsolomon.Reconstruct() is used to reconstruct the missing parts.
+func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string, size int64, blockSize int64, dataBlocks int, parityBlocks int, algo string) (checkSums []string, err error) {
+	var offset int64
+	remainingSize := size
+
+	// Hash for bitrot protection.
+	hashWriters := newHashWriters(len(outDatedDisks), bitRotAlgo)
+
+	for remainingSize > 0 {
+		curBlockSize := blockSize
+		if remainingSize < curBlockSize {
+			curBlockSize = remainingSize
+		}
+
+		// Calculate the block size that needs to be read from each disk.
+		curEncBlockSize := getChunkSize(curBlockSize, dataBlocks)
+
+		// Memory for reading data from disks and reconstructing missing data using erasure coding.
+		enBlocks := make([][]byte, len(latestDisks))
+
+		// Read data from the latest disks.
+		// FIXME: no need to read from all the disks. dataBlocks+1 is enough.
+		for index, disk := range latestDisks {
+			if disk == nil {
+				continue
+			}
+			enBlocks[index] = make([]byte, curEncBlockSize)
+			_, err := disk.ReadFile(volume, path, offset, enBlocks[index])
+			if err != nil {
+				enBlocks[index] = nil
+			}
+		}
+
+		// Reconstruct missing data.
+		err := decodeData(enBlocks, dataBlocks, parityBlocks)
+		if err != nil {
+			return nil, err
+		}
+
+		// Write to the healPath file.
+		for index, disk := range outDatedDisks {
+			if disk == nil {
+				continue
+			}
+			err := disk.AppendFile(healBucket, healPath, enBlocks[index])
+			if err != nil {
+				return nil, traceError(err)
+			}
+			hashWriters[index].Write(enBlocks[index])
+		}
+		remainingSize -= curBlockSize
+		offset += curEncBlockSize
+	}
+
+	// Checksums for the bit rot.
+	checkSums = make([]string, len(outDatedDisks))
+	for index, disk := range outDatedDisks {
+		if disk == nil {
+			continue
+		}
+		checkSums[index] = hex.EncodeToString(hashWriters[index].Sum(nil))
+	}
+	return checkSums, nil
+}

cmd/erasure-healfile_test.go

@@ -0,0 +1,129 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/rand"
+	"os"
+	"path"
+	"testing"
+
+	humanize "github.com/dustin/go-humanize"
+)
+
+// Test erasureHealFile()
+func TestErasureHealFile(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 1MiB with random data.
+	data := make([]byte, 1*humanize.MiByte)
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Create a test file.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	latest := make([]StorageAPI, len(disks))   // Slice of latest disks
+	outDated := make([]StorageAPI, len(disks)) // Slice of outdated disks
+
+	// Test case when one part needs to be healed.
+	dataPath := path.Join(setup.diskPaths[0], "testbucket", "testobject1")
+	err = os.Remove(dataPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	copy(latest, disks)
+	latest[0] = nil
+	outDated[0] = disks[0]
+
+	healCheckSums, err := erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*humanize.MiByte, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Checksum of the healed file should match.
+	if checkSums[0] != healCheckSums[0] {
+		t.Error("Healing failed, data does not match.")
+	}
+
+	// Test case when parityBlocks number of disks need to be healed.
+	// Should succeed.
+	copy(latest, disks)
+	for index := 0; index < parityBlocks; index++ {
+		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
+		err = os.Remove(dataPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		latest[index] = nil
+		outDated[index] = disks[index]
+	}
+
+	healCheckSums, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*humanize.MiByte, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Checksums of the healed files should match.
+	for index := 0; index < parityBlocks; index++ {
+		if checkSums[index] != healCheckSums[index] {
+			t.Error("Healing failed, data does not match.")
+		}
+	}
+	for index := dataBlocks; index < len(disks); index++ {
+		if healCheckSums[index] != "" {
+			t.Errorf("expected healCheckSums[%d] to be empty", index)
+		}
+	}
+
+	// Test case when parityBlocks+1 number of disks need to be healed.
+	// Should fail.
+	copy(latest, disks)
+	for index := 0; index < parityBlocks+1; index++ {
+		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
+		err = os.Remove(dataPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		latest[index] = nil
+		outDated[index] = disks[index]
+	}
+	_, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*humanize.MiByte, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err == nil {
+		t.Error("Expected erasureHealFile() to fail when the number of available disks <= parityBlocks")
+	}
+}

cmd/erasure-readfile.go

@@ -0,0 +1,350 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/hex"
+	"errors"
+	"io"
+	"sync"
+
+	"github.com/klauspost/reedsolomon"
+	"github.com/minio/minio/pkg/bpool"
+)
+
+// isSuccessDecodeBlocks - do we have all the blocks to be
+// successfully decoded?. Input encoded blocks ordered matrix.
+func isSuccessDecodeBlocks(enBlocks [][]byte, dataBlocks int) bool {
+	// Count number of data and parity blocks that were read.
+	var successDataBlocksCount = 0
+	var successParityBlocksCount = 0
+	for index := range enBlocks {
+		if enBlocks[index] == nil {
+			continue
+		}
+		// block index lesser than data blocks, update data block count.
+		if index < dataBlocks {
+			successDataBlocksCount++
+			continue
+		} // else { // update parity block count.
+		successParityBlocksCount++
+	}
+	// Returns true if we have atleast dataBlocks parity.
+	return successDataBlocksCount == dataBlocks || successDataBlocksCount+successParityBlocksCount >= dataBlocks
+}
+
+// isSuccessDataBlocks - do we have all the data blocks?
+// Input encoded blocks ordered matrix.
+func isSuccessDataBlocks(enBlocks [][]byte, dataBlocks int) bool {
+	// Count number of data blocks that were read.
+	var successDataBlocksCount = 0
+	for index := range enBlocks[:dataBlocks] {
+		if enBlocks[index] == nil {
+			continue
+		}
+		// block index lesser than data blocks, update data block count.
+		if index < dataBlocks {
+			successDataBlocksCount++
+		}
+	}
+	// Returns true if we have atleast the dataBlocks.
+	return successDataBlocksCount >= dataBlocks
+}
+
+// Return readable disks slice from which we can read parallelly.
+func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDisks []StorageAPI, nextIndex int, err error) {
+	readDisks = make([]StorageAPI, len(orderedDisks))
+	dataDisks := 0
+	parityDisks := 0
+	// Count already read data and parity chunks.
+	for i := 0; i < index; i++ {
+		if orderedDisks[i] == nil {
+			continue
+		}
+		if i < dataBlocks {
+			dataDisks++
+		} else {
+			parityDisks++
+		}
+	}
+
+	// Sanity checks - we should never have this situation.
+	if dataDisks == dataBlocks {
+		return nil, 0, traceError(errUnexpected)
+	}
+	if dataDisks+parityDisks >= dataBlocks {
+		return nil, 0, traceError(errUnexpected)
+	}
+
+	// Find the disks from which next set of parallel reads should happen.
+	for i := index; i < len(orderedDisks); i++ {
+		if orderedDisks[i] == nil {
+			continue
+		}
+		if i < dataBlocks {
+			dataDisks++
+		} else {
+			parityDisks++
+		}
+		readDisks[i] = orderedDisks[i]
+		if dataDisks == dataBlocks {
+			return readDisks, i + 1, nil
+		} else if dataDisks+parityDisks == dataBlocks {
+			return readDisks, i + 1, nil
+		}
+	}
+	return nil, 0, traceError(errXLReadQuorum)
+}
+
+// parallelRead - reads chunks in parallel from the disks specified in []readDisks.
+func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []StorageAPI, enBlocks [][]byte, blockOffset int64, curChunkSize int64, bitRotVerify func(diskIndex int) bool, pool *bpool.BytePool) {
+	// WaitGroup to synchronise the read go-routines.
+	wg := &sync.WaitGroup{}
+
+	// Read disks in parallel.
+	for index := range readDisks {
+		if readDisks[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		// Reads chunk from readDisk[index] in routine.
+		go func(index int) {
+			defer wg.Done()
+
+			// Verify bit rot for the file on this disk.
+			if !bitRotVerify(index) {
+				// So that we don't read from this disk for the next block.
+				orderedDisks[index] = nil
+				return
+			}
+
+			buf, err := pool.Get()
+			if err != nil {
+				errorIf(err, "unable to get buffer from byte pool")
+				orderedDisks[index] = nil
+				return
+			}
+			buf = buf[:curChunkSize]
+
+			_, err = readDisks[index].ReadFile(volume, path, blockOffset, buf)
+			if err != nil {
+				orderedDisks[index] = nil
+				return
+			}
+			enBlocks[index] = buf
+		}(index)
+	}
+
+	// Waiting for first routines to finish.
+	wg.Wait()
+}
+
+// erasureReadFile - read bytes from erasure coded files and writes to given writer.
+// Erasure coded files are read block by block as per given erasureInfo and data chunks
+// are decoded into a data block. Data block is trimmed for given offset and length,
+// then written to given writer. This function also supports bit-rot detection by
+// verifying checksum of individual block's checksum.
+func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path string, offset int64, length int64, totalLength int64, blockSize int64, dataBlocks int, parityBlocks int, checkSums []string, algo string, pool *bpool.BytePool) (int64, error) {
+	// Offset and length cannot be negative.
+	if offset < 0 || length < 0 {
+		return 0, traceError(errUnexpected)
+	}
+
+	// Can't request more data than what is available.
+	if offset+length > totalLength {
+		return 0, traceError(errUnexpected)
+	}
+
+	// chunkSize is the amount of data that needs to be read from each disk at a time.
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+
+	// bitRotVerify verifies if the file on a particular disk doesn't have bitrot
+	// by verifying the hash of the contents of the file.
+	bitRotVerify := func() func(diskIndex int) bool {
+		verified := make([]bool, len(disks))
+		// Return closure so that we have reference to []verified and
+		// not recalculate the hash on it every time the function is
+		// called for the same disk.
+		return func(diskIndex int) bool {
+			if verified[diskIndex] {
+				// Already validated.
+				return true
+			}
+			// Is this a valid block?
+			isValid := isValidBlock(disks[diskIndex], volume, path, checkSums[diskIndex], algo)
+			verified[diskIndex] = isValid
+			return isValid
+		}
+	}()
+
+	// Total bytes written to writer
+	bytesWritten := int64(0)
+
+	startBlock := offset / blockSize
+	endBlock := (offset + length) / blockSize
+
+	// curChunkSize = chunk size for the current block in the for loop below.
+	// curBlockSize = block size for the current block in the for loop below.
+	// curChunkSize and curBlockSize can change for the last block if totalLength%blockSize != 0
+	curChunkSize := chunkSize
+	curBlockSize := blockSize
+
+	// For each block, read chunk from each disk. If we are able to read all the data disks then we don't
+	// need to read parity disks. If one of the data disk is missing we need to read DataBlocks+1 number
+	// of disks. Once read, we Reconstruct() missing data if needed and write it to the given writer.
+	for block := startBlock; block <= endBlock; block++ {
+		// Mark all buffers as unused at the start of the loop so that the buffers
+		// can be reused.
+		pool.Reset()
+
+		// Each element of enBlocks holds curChunkSize'd amount of data read from its corresponding disk.
+		enBlocks := make([][]byte, len(disks))
+
+		if ((offset + bytesWritten) / blockSize) == (totalLength / blockSize) {
+			// This is the last block for which curBlockSize and curChunkSize can change.
+			// For ex. if totalLength is 15M and blockSize is 10MB, curBlockSize for
+			// the last block should be 5MB.
+			curBlockSize = totalLength % blockSize
+			curChunkSize = getChunkSize(curBlockSize, dataBlocks)
+		}
+
+		// NOTE: That for the offset calculation we have to use chunkSize and
+		// not curChunkSize. If we use curChunkSize for offset calculation
+		// then it can result in wrong offset for the last block.
+		blockOffset := block * chunkSize
+
+		// nextIndex - index from which next set of parallel reads
+		// should happen.
+		nextIndex := 0
+
+		for {
+			// readDisks - disks from which we need to read in parallel.
+			var readDisks []StorageAPI
+			var err error
+			// get readable disks slice from which we can read parallelly.
+			readDisks, nextIndex, err = getReadDisks(disks, nextIndex, dataBlocks)
+			if err != nil {
+				return bytesWritten, err
+			}
+			// Issue a parallel read across the disks specified in readDisks.
+			parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, bitRotVerify, pool)
+			if isSuccessDecodeBlocks(enBlocks, dataBlocks) {
+				// If enough blocks are available to do rs.Reconstruct()
+				break
+			}
+			if nextIndex == len(disks) {
+				// No more disks to read from.
+				return bytesWritten, traceError(errXLReadQuorum)
+			}
+			// We do not have enough enough data blocks to reconstruct the data
+			// hence continue the for-loop till we have enough data blocks.
+		}
+
+		// If we have all the data blocks no need to decode, continue to write.
+		if !isSuccessDataBlocks(enBlocks, dataBlocks) {
+			// Reconstruct the missing data blocks.
+			if err := decodeData(enBlocks, dataBlocks, parityBlocks); err != nil {
+				return bytesWritten, err
+			}
+		}
+
+		// Offset in enBlocks from where data should be read from.
+		enBlocksOffset := int64(0)
+
+		// Total data to be read from enBlocks.
+		enBlocksLength := curBlockSize
+
+		// If this is the start block then enBlocksOffset might not be 0.
+		if block == startBlock {
+			enBlocksOffset = offset % blockSize
+			enBlocksLength -= enBlocksOffset
+		}
+
+		remaining := length - bytesWritten
+		if remaining < enBlocksLength {
+			// We should not send more data than what was requested.
+			enBlocksLength = remaining
+		}
+
+		// Write data blocks.
+		n, err := writeDataBlocks(writer, enBlocks, dataBlocks, enBlocksOffset, enBlocksLength)
+		if err != nil {
+			return bytesWritten, err
+		}
+
+		// Update total bytes written.
+		bytesWritten += n
+
+		if bytesWritten == length {
+			// Done writing all the requested data.
+			break
+		}
+	}
+
+	// Success.
+	return bytesWritten, nil
+}
+
+// isValidBlock - calculates the checksum hash for the block and
+// validates if its correct returns true for valid cases, false otherwise.
+func isValidBlock(disk StorageAPI, volume, path, checkSum, checkSumAlgo string) (ok bool) {
+	// Disk is not available, not a valid block.
+	if disk == nil {
+		return false
+	}
+	// Checksum not available, not a valid block.
+	if checkSum == "" {
+		return false
+	}
+	// Read everything for a given block and calculate hash.
+	hashWriter := newHash(checkSumAlgo)
+	hashBytes, err := hashSum(disk, volume, path, hashWriter)
+	if err != nil {
+		errorIf(err, "Unable to calculate checksum %s/%s", volume, path)
+		return false
+	}
+	return hex.EncodeToString(hashBytes) == checkSum
+}
+
+// decodeData - decode encoded blocks.
+func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
+	// Initialized reedsolomon.
+	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
+	if err != nil {
+		return traceError(err)
+	}
+
+	// Reconstruct encoded blocks.
+	err = rs.Reconstruct(enBlocks)
+	if err != nil {
+		return traceError(err)
+	}
+
+	// Verify reconstructed blocks (parity).
+	ok, err := rs.Verify(enBlocks)
+	if err != nil {
+		return traceError(err)
+	}
+	if !ok {
+		// Blocks cannot be reconstructed, corrupted data.
+		err = errors.New("Verification failed after reconstruction, data likely corrupted")
+		return traceError(err)
+	}
+
+	// Success.
+	return nil
+}

cmd/erasure-readfile_test.go

@@ -0,0 +1,471 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"math/rand"
+	"testing"
+	"time"
+
+	"reflect"
+
+	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/pkg/bpool"
+)
+
+// Tests getReadDisks which returns readable disks slice from which we can
+// read parallelly.
+func testGetReadDisks(t *testing.T, xl *xlObjects) {
+	d := xl.storageDisks
+	testCases := []struct {
+		index     int          // index argument for getReadDisks
+		argDisks  []StorageAPI // disks argument for getReadDisks
+		retDisks  []StorageAPI // disks return value from getReadDisks
+		nextIndex int          // return value from getReadDisks
+		err       error        // error return value from getReadDisks
+	}{
+		// Test case - 1.
+		// When all disks are available, should return data disks.
+		{
+			0,
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
+			8,
+			nil,
+		},
+		// Test case - 2.
+		// If a parity disk is down, should return all data disks.
+		{
+			0,
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], nil, d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
+			8,
+			nil,
+		},
+		// Test case - 3.
+		// If a data disk is down, should return 7 data and 1 parity.
+		{
+			0,
+			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], nil, nil, nil, nil, nil, nil, nil},
+			9,
+			nil,
+		},
+		// Test case - 4.
+		// If 7 data disks are down, should return 1 data and 7 parity.
+		{
+			0,
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], nil},
+			15,
+			nil,
+		},
+		// Test case - 5.
+		// When 2 disks fail during parallelRead, next call to getReadDisks should return 3 disks
+		{
+			8,
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, d[8], d[9], nil, nil, nil, nil, nil, nil},
+			10,
+			nil,
+		},
+		// Test case - 6.
+		// If 2 disks again fail from the 3 disks returned previously, return next 2 disks
+		{
+			11,
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, d[11], nil, nil, nil, nil},
+			12,
+			nil,
+		},
+		// Test case - 7.
+		// No more disks are available for read, return error
+		{
+			13,
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], nil, nil, nil, nil, nil},
+			nil,
+			0,
+			errXLReadQuorum,
+		},
+	}
+
+	for i, test := range testCases {
+		disks, nextIndex, err := getReadDisks(test.argDisks, test.index, xl.dataBlocks)
+		if errorCause(err) != test.err {
+			t.Errorf("test-case %d - expected error : %s, got : %s", i+1, test.err, err)
+			continue
+		}
+		if test.nextIndex != nextIndex {
+			t.Errorf("test-case %d - expected nextIndex: %d, got : %d", i+1, test.nextIndex, nextIndex)
+			continue
+		}
+		if !reflect.DeepEqual(test.retDisks, disks) {
+			t.Errorf("test-case %d : incorrect disks returned. expected %+v, got %+v", i+1, test.retDisks, disks)
+			continue
+		}
+	}
+}
+
+// Test getOrderedDisks which returns ordered slice of disks from their
+// actual distribution.
+func testGetOrderedDisks(t *testing.T, xl *xlObjects) {
+	disks := xl.storageDisks
+	distribution := []int{16, 14, 12, 10, 8, 6, 4, 2, 1, 3, 5, 7, 9, 11, 13, 15}
+	orderedDisks := getOrderedDisks(distribution, disks)
+	// From the "distribution" above you can notice that:
+	// 1st data block is in the 9th disk (i.e distribution index 8)
+	// 2nd data block is in the 8th disk (i.e distribution index 7) and so on.
+	if orderedDisks[0] != disks[8] ||
+		orderedDisks[1] != disks[7] ||
+		orderedDisks[2] != disks[9] ||
+		orderedDisks[3] != disks[6] ||
+		orderedDisks[4] != disks[10] ||
+		orderedDisks[5] != disks[5] ||
+		orderedDisks[6] != disks[11] ||
+		orderedDisks[7] != disks[4] ||
+		orderedDisks[8] != disks[12] ||
+		orderedDisks[9] != disks[3] ||
+		orderedDisks[10] != disks[13] ||
+		orderedDisks[11] != disks[2] ||
+		orderedDisks[12] != disks[14] ||
+		orderedDisks[13] != disks[1] ||
+		orderedDisks[14] != disks[15] ||
+		orderedDisks[15] != disks[0] {
+		t.Errorf("getOrderedDisks returned incorrect order.")
+	}
+}
+
+// Test for isSuccessDataBlocks and isSuccessDecodeBlocks.
+func TestIsSuccessBlocks(t *testing.T) {
+	dataBlocks := 8
+	testCases := []struct {
+		enBlocks      [][]byte // data and parity blocks.
+		successData   bool     // expected return value of isSuccessDataBlocks()
+		successDecode bool     // expected return value of isSuccessDecodeBlocks()
+	}{
+		{
+			// When all data and partity blocks are available.
+			[][]byte{
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			true,
+			true,
+		},
+		{
+			// When one data block is not available.
+			[][]byte{
+				nil, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// When one data and all parity are available, enough for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// When all data disks are not available, enough for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// Not enough disks for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			false,
+		},
+	}
+
+	for i, test := range testCases {
+		got := isSuccessDataBlocks(test.enBlocks, dataBlocks)
+		if test.successData != got {
+			t.Errorf("test-case %d : expected %v got %v", i+1, test.successData, got)
+		}
+		got = isSuccessDecodeBlocks(test.enBlocks, dataBlocks)
+		if test.successDecode != got {
+			t.Errorf("test-case %d : expected %v got %v", i+1, test.successDecode, got)
+		}
+	}
+}
+
+// Wrapper function for testGetReadDisks, testGetOrderedDisks.
+func TestErasureReadUtils(t *testing.T) {
+	nDisks := 16
+	disks, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	endpoints, err := parseStorageEndpoints(disks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	objLayer, _, err := initObjectLayer(endpoints)
+	if err != nil {
+		removeRoots(disks)
+		t.Fatal(err)
+	}
+	defer removeRoots(disks)
+	xl := objLayer.(*xlObjects)
+	testGetReadDisks(t, xl)
+	testGetOrderedDisks(t, xl)
+}
+
+// Simulates a faulty disk for ReadFile()
+type ReadDiskDown struct {
+	*posix
+}
+
+func (r ReadDiskDown) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
+	return 0, errFaultyDisk
+}
+
+func TestErasureReadFileDiskFail(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 1humanize.MiByte with random data.
+	data := make([]byte, 1*humanize.MiByte)
+	length := int64(len(data))
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a test file to read from.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
+	}
+
+	// create byte pool which will be used by erasureReadFile for
+	// reading from disks and erasure decoding.
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+	pool := bpool.NewBytePool(chunkSize, len(disks))
+
+	buf := &bytes.Buffer{}
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(buf.Bytes(), data) {
+		t.Error("Contents of the erasure coded file differs")
+	}
+
+	// 2 disks down. Read should succeed.
+	disks[4] = ReadDiskDown{disks[4].(*posix)}
+	disks[5] = ReadDiskDown{disks[5].(*posix)}
+
+	buf.Reset()
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(buf.Bytes(), data) {
+		t.Error("Contents of the erasure coded file differs")
+	}
+
+	// 4 more disks down. 6 disks down in total. Read should succeed.
+	disks[6] = ReadDiskDown{disks[6].(*posix)}
+	disks[8] = ReadDiskDown{disks[8].(*posix)}
+	disks[9] = ReadDiskDown{disks[9].(*posix)}
+	disks[11] = ReadDiskDown{disks[11].(*posix)}
+
+	buf.Reset()
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(buf.Bytes(), data) {
+		t.Error("Contents of the erasure coded file differs")
+	}
+
+	// 2 more disk down. 8 disks down in total. Read should fail.
+	disks[12] = ReadDiskDown{disks[12].(*posix)}
+	disks[13] = ReadDiskDown{disks[13].(*posix)}
+	buf.Reset()
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if errorCause(err) != errXLReadQuorum {
+		t.Fatal("expected errXLReadQuorum error")
+	}
+}
+
+func TestErasureReadFileOffsetLength(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(1 * humanize.MiByte)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 5humanize.MiByte with random data.
+	data := make([]byte, 5*humanize.MiByte)
+	length := int64(len(data))
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a test file to read from.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
+	}
+
+	testCases := []struct {
+		offset, length int64
+	}{
+		// Full file.
+		{0, length},
+		// Read nothing.
+		{length, 0},
+		// 2nd block.
+		{blockSize, blockSize},
+		// Test cases for random offsets and lengths.
+		{blockSize - 1, 2},
+		{blockSize - 1, blockSize + 1},
+		{blockSize + 1, blockSize - 1},
+		{blockSize + 1, blockSize},
+		{blockSize + 1, blockSize + 1},
+		{blockSize*2 - 1, blockSize + 1},
+		{length - 1, 1},
+		{length - blockSize, blockSize},
+		{length - blockSize - 1, blockSize},
+		{length - blockSize - 1, blockSize + 1},
+	}
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+	pool := bpool.NewBytePool(chunkSize, len(disks))
+
+	// Compare the data read from file with "data" byte array.
+	for i, testCase := range testCases {
+		expected := data[testCase.offset:(testCase.offset + testCase.length)]
+		buf := &bytes.Buffer{}
+		_, err = erasureReadFile(buf, disks, "testbucket", "testobject", testCase.offset, testCase.length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		got := buf.Bytes()
+		if !bytes.Equal(expected, got) {
+			t.Errorf("Test %d : read data is different from what was expected", i+1)
+		}
+	}
+}
+
+// Test erasureReadFile with random offset and lengths.
+// This test is t.Skip()ed as it a long time to run, hence should be run
+// explicitly after commenting out t.Skip()
+func TestErasureReadFileRandomOffsetLength(t *testing.T) {
+	// Comment the following line to run this test.
+	t.SkipNow()
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(1 * humanize.MiByte)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 5MiB with random data.
+	data := make([]byte, 5*humanize.MiByte)
+	length := int64(len(data))
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 10000 iterations with random offsets and lengths.
+	iterations := 10000
+
+	// Create a test file to read from.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
+	}
+
+	// To generate random offset/length.
+	r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
+
+	// create pool buffer which will be used by erasureReadFile for
+	// reading from disks and erasure decoding.
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+	pool := bpool.NewBytePool(chunkSize, len(disks))
+
+	buf := &bytes.Buffer{}
+
+	// Verify erasureReadFile() for random offsets and lengths.
+	for i := 0; i < iterations; i++ {
+		offset := r.Int63n(length)
+		readLen := r.Int63n(length - offset)
+
+		expected := data[offset : offset+readLen]
+
+		_, err = erasureReadFile(buf, disks, "testbucket", "testobject", offset, readLen, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+		if err != nil {
+			t.Fatal(err, offset, readLen)
+		}
+		got := buf.Bytes()
+		if !bytes.Equal(expected, got) {
+			t.Fatalf("read data is different from what was expected, offset=%d length=%d", offset, readLen)
+		}
+		buf.Reset()
+	}
+}

cmd/erasure-utils.go

@@ -0,0 +1,197 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"hash"
+	"io"
+	"sync"
+
+	"github.com/klauspost/reedsolomon"
+	"github.com/minio/blake2b-simd"
+)
+
+// newHashWriters - inititialize a slice of hashes for the disk count.
+func newHashWriters(diskCount int, algo string) []hash.Hash {
+	hashWriters := make([]hash.Hash, diskCount)
+	for index := range hashWriters {
+		hashWriters[index] = newHash(algo)
+	}
+	return hashWriters
+}
+
+// newHash - gives you a newly allocated hash depending on the input algorithm.
+func newHash(algo string) hash.Hash {
+	switch algo {
+	case "blake2b":
+		return blake2b.New512()
+	// Add new hashes here.
+	default:
+		// Default to blake2b.
+		return blake2b.New512()
+	}
+}
+
+// Hash buffer pool is a pool of reusable
+// buffers used while checksumming a stream.
+var hashBufferPool = sync.Pool{
+	New: func() interface{} {
+		b := make([]byte, readSizeV1)
+		return &b
+	},
+}
+
+// hashSum calculates the hash of the entire path and returns.
+func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
+	// Fetch staging a new staging buffer from the pool.
+	bufp := hashBufferPool.Get().(*[]byte)
+	defer hashBufferPool.Put(bufp)
+
+	// Copy entire buffer to writer.
+	if err := copyBuffer(writer, disk, volume, path, *bufp); err != nil {
+		return nil, err
+	}
+
+	// Return the final hash sum.
+	return writer.Sum(nil), nil
+}
+
+// getDataBlockLen - get length of data blocks from encoded blocks.
+func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
+	size := 0
+	// Figure out the data block length.
+	for _, block := range enBlocks[:dataBlocks] {
+		size += len(block)
+	}
+	return size
+}
+
+// Writes all the data blocks from encoded blocks until requested
+// outSize length. Provides a way to skip bytes until the offset.
+func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
+	// Offset and out size cannot be negative.
+	if offset < 0 || length < 0 {
+		return 0, traceError(errUnexpected)
+	}
+
+	// Do we have enough blocks?
+	if len(enBlocks) < dataBlocks {
+		return 0, traceError(reedsolomon.ErrTooFewShards)
+	}
+
+	// Do we have enough data?
+	if int64(getDataBlockLen(enBlocks, dataBlocks)) < length {
+		return 0, traceError(reedsolomon.ErrShortData)
+	}
+
+	// Counter to decrement total left to write.
+	write := length
+
+	// Counter to increment total written.
+	totalWritten := int64(0)
+
+	// Write all data blocks to dst.
+	for _, block := range enBlocks[:dataBlocks] {
+		// Skip blocks until we have reached our offset.
+		if offset >= int64(len(block)) {
+			// Decrement offset.
+			offset -= int64(len(block))
+			continue
+		} else {
+			// Skip until offset.
+			block = block[offset:]
+
+			// Reset the offset for next iteration to read everything
+			// from subsequent blocks.
+			offset = 0
+		}
+		// We have written all the blocks, write the last remaining block.
+		if write < int64(len(block)) {
+			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
+			if err != nil {
+				return 0, traceError(err)
+			}
+			totalWritten += n
+			break
+		}
+		// Copy the block.
+		n, err := io.Copy(dst, bytes.NewReader(block))
+		if err != nil {
+			return 0, traceError(err)
+		}
+
+		// Decrement output size.
+		write -= n
+
+		// Increment written.
+		totalWritten += n
+	}
+
+	// Success.
+	return totalWritten, nil
+}
+
+// chunkSize is roughly BlockSize/DataBlocks.
+// chunkSize is calculated such that chunkSize*DataBlocks accommodates BlockSize bytes.
+// So chunkSize*DataBlocks can be slightly larger than BlockSize if BlockSize is not divisible by
+// DataBlocks. The extra space will have 0-padding.
+func getChunkSize(blockSize int64, dataBlocks int) int64 {
+	return (blockSize + int64(dataBlocks) - 1) / int64(dataBlocks)
+}
+
+// copyBuffer - copies from disk, volume, path to input writer until either EOF
+// is reached at volume, path or an error occurs. A success copyBuffer returns
+// err == nil, not err == EOF. Because copyBuffer is defined to read from path
+// until EOF. It does not treat an EOF from ReadFile an error to be reported.
+// Additionally copyBuffer stages through the provided buffer; otherwise if it
+// has zero length, returns error.
+func copyBuffer(writer io.Writer, disk StorageAPI, volume string, path string, buf []byte) error {
+	// Error condition of zero length buffer.
+	if buf != nil && len(buf) == 0 {
+		return errors.New("empty buffer in readBuffer")
+	}
+
+	// Starting offset for Reading the file.
+	startOffset := int64(0)
+
+	// Read until io.EOF.
+	for {
+		n, err := disk.ReadFile(volume, path, startOffset, buf)
+		if n > 0 {
+			m, wErr := writer.Write(buf[:n])
+			if wErr != nil {
+				return wErr
+			}
+			if int64(m) != n {
+				return io.ErrShortWrite
+			}
+		}
+		if err != nil {
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				break
+			}
+			return err
+		}
+		// Progress the offset.
+		startOffset += n
+	}
+
+	// Success.
+	return nil
+}

cmd/erasure-utils_test.go

@@ -0,0 +1,160 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"testing"
+)
+
+// Test validates the number hash writers returned.
+func TestNewHashWriters(t *testing.T) {
+	diskNum := 8
+	hashWriters := newHashWriters(diskNum, bitRotAlgo)
+	if len(hashWriters) != diskNum {
+		t.Errorf("Expected %d hashWriters, but instead got %d", diskNum, len(hashWriters))
+	}
+
+}
+
+// Tests validate the output of getChunkSize.
+func TestGetChunkSize(t *testing.T) {
+	// Refer to comments on getChunkSize() for details.
+	testCases := []struct {
+		blockSize  int64
+		dataBlocks int
+		// expected result.
+		expectedChunkSize int64
+	}{
+		{
+			10,
+			10,
+			1,
+		},
+		{
+			10,
+			11,
+			1,
+		},
+		{
+			10,
+			9,
+			2,
+		},
+	}
+	// Verify getChunkSize() for the test cases.
+	for i, testCase := range testCases {
+		got := getChunkSize(testCase.blockSize, testCase.dataBlocks)
+		if testCase.expectedChunkSize != got {
+			t.Errorf("Test %d : expected=%d got=%d", i+1, testCase.expectedChunkSize, got)
+		}
+	}
+}
+
+// TestCopyBuffer - Tests validate the result and errors produced when `copyBuffer` is called with sample inputs.
+func TestCopyBuffer(t *testing.T) {
+	// create posix test setup
+	disk, diskPath, err := newPosixTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer removeAll(diskPath)
+
+	volume := "success-vol"
+	// Setup test environment.
+	if err = disk.MakeVol(volume); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+
+	// creating io.Writer for the input to copyBuffer.
+	buffers := []*bytes.Buffer{
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+	}
+
+	testFile := "testFile"
+	testContent := []byte("hello, world")
+	err = disk.AppendFile(volume, testFile, testContent)
+	if err != nil {
+		t.Fatalf("AppendFile failed: <ERROR> %s", err)
+	}
+
+	testCases := []struct {
+		writer io.Writer
+		disk   StorageAPI
+		volume string
+		path   string
+		buf    []byte
+
+		expectedResult []byte
+		// flag to indicate whether test case should pass.
+		shouldPass  bool
+		expectedErr error
+	}{
+		// Test case - 1.
+		// case with empty buffer.
+		{nil, nil, "", "", []byte{}, nil, false, errors.New("empty buffer in readBuffer")},
+		// Test case - 2.
+		// Test case with empty volume.
+		{buffers[0], disk, "", "", make([]byte, 5), nil, false, errInvalidArgument},
+		// Test case - 3.
+		// Test case with non existent volume.
+		{buffers[0], disk, "abc", "", make([]byte, 5), nil, false, errVolumeNotFound},
+		// Test case - 4.
+		// Test case with empty filename/path.
+		{buffers[0], disk, volume, "", make([]byte, 5), nil, false, errIsNotRegular},
+		// Test case - 5.
+		// Test case with non existent file name.
+		{buffers[0], disk, volume, "abcd", make([]byte, 5), nil, false, errFileNotFound},
+		// Test case - 6.
+		// Test case where the writer returns EOF.
+		{NewEOFWriter(buffers[0], 3), disk, volume, testFile, make([]byte, 5), nil, false, io.EOF},
+		// Test case - 7.
+		// Test case to produce io.ErrShortWrite, the TruncateWriter returns after writing 3 bytes.
+		{TruncateWriter(buffers[1], 3), disk, volume, testFile, make([]byte, 5), nil, false, io.ErrShortWrite},
+		// Teset case - 8.
+		// Valid case, expected to read till EOF and write the contents into the writer.
+		{buffers[2], disk, volume, testFile, make([]byte, 5), testContent, true, nil},
+	}
+	// iterate over the test cases and call copy Buffer with data.
+	for i, testCase := range testCases {
+		actualErr := copyBuffer(testCase.writer, testCase.disk, testCase.volume, testCase.path, testCase.buf)
+
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
+		}
+
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if testCase.expectedErr.Error() != actualErr.Error() {
+				t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
+			}
+		}
+		// test passed as expected, asserting the result.
+		if actualErr == nil && testCase.shouldPass {
+			if !bytes.Equal(testCase.expectedResult, buffers[2].Bytes()) {
+				t.Errorf("Test %d: copied buffer differs from the expected one.", i+1)
+			}
+		}
+	}
+}

cmd/erasure_test.go

@@ -0,0 +1,184 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+)
+
+// mustEncodeData - encodes data slice and provides encoded 2 dimensional slice.
+func mustEncodeData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData, err := encodeData(data, dataBlocks, parityBlocks)
+	if err != nil {
+		// Upon failure panic this function.
+		panic(err)
+	}
+	return encodedData
+}
+
+// Generates good encoded data with one parity block and data block missing.
+func getGoodEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	encodedData[3] = nil
+	encodedData[1] = nil
+	return encodedData
+}
+
+// Generates bad encoded data with one parity block and data block with garbage data.
+func getBadEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	encodedData[3] = []byte("garbage")
+	encodedData[1] = []byte("garbage")
+	return encodedData
+}
+
+// Generates encoded data with all data blocks missing.
+func getMissingData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	for i := 0; i < dataBlocks+1; i++ {
+		encodedData[i] = nil
+	}
+	return encodedData
+}
+
+// Generates encoded data with less number of blocks than expected data blocks.
+func getInsufficientData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	// Return half of the data.
+	return encodedData[:dataBlocks/2]
+}
+
+// Represents erasure encoding matrix dataBlocks and paritBlocks.
+type encodingMatrix struct {
+	dataBlocks   int
+	parityBlocks int
+}
+
+// List of encoding matrices the tests will run on.
+var encodingMatrices = []encodingMatrix{
+	{3, 3}, // 3 data, 3 parity blocks.
+	{4, 4}, // 4 data, 4 parity blocks.
+	{5, 5}, // 5 data, 5 parity blocks.
+	{6, 6}, // 6 data, 6 parity blocks.
+	{7, 7}, // 7 data, 7 parity blocks.
+	{8, 8}, // 8 data, 8 parity blocks.
+}
+
+// Tests erasure decoding functionality for various types of inputs.
+func TestErasureDecode(t *testing.T) {
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+
+	// List of decoding cases
+	// - validates bad encoded data.
+	// - validates good encoded data.
+	// - validates insufficient data.
+	testDecodeCases := []struct {
+		enFn       func([]byte, int, int) [][]byte
+		shouldPass bool
+	}{
+		// Generates bad encoded data.
+		{
+			enFn:       getBadEncodedData,
+			shouldPass: false,
+		},
+		// Generates good encoded data.
+		{
+			enFn:       getGoodEncodedData,
+			shouldPass: true,
+		},
+		// Generates missing data.
+		{
+			enFn:       getMissingData,
+			shouldPass: false,
+		},
+		// Generates short data.
+		{
+			enFn:       getInsufficientData,
+			shouldPass: false,
+		},
+	}
+
+	// Validates all decode tests.
+	for i, testCase := range testDecodeCases {
+		for _, encodingMatrix := range encodingMatrices {
+
+			// Encoding matrix.
+			dataBlocks := encodingMatrix.dataBlocks
+			parityBlocks := encodingMatrix.parityBlocks
+
+			// Data block size.
+			blockSize := len(data)
+
+			// Generates encoded data based on type of testCase function.
+			encodedData := testCase.enFn(data, dataBlocks, parityBlocks)
+
+			// Decodes the data.
+			err := decodeData(encodedData, dataBlocks, parityBlocks)
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
+			}
+
+			// Proceed to extract the data blocks.
+			decodedDataWriter := new(bytes.Buffer)
+			_, err = writeDataBlocks(decodedDataWriter, encodedData, dataBlocks, 0, int64(blockSize))
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
+			}
+
+			// Validate if decoded data is what we expected.
+			if bytes.Equal(decodedDataWriter.Bytes(), data) != testCase.shouldPass {
+				err := errUnexpected
+				t.Errorf("Test %d: Expected to pass by failed instead %s", i+1, err)
+			}
+		}
+	}
+}
+
+// Setup for erasureCreateFile and erasureReadFile tests.
+type erasureTestSetup struct {
+	dataBlocks   int
+	parityBlocks int
+	blockSize    int64
+	diskPaths    []string
+	disks        []StorageAPI
+}
+
+// Removes the temporary disk directories.
+func (e erasureTestSetup) Remove() {
+	for _, path := range e.diskPaths {
+		removeAll(path)
+	}
+}
+
+// Returns an initialized setup for erasure tests.
+func newErasureTestSetup(dataBlocks int, parityBlocks int, blockSize int64) (*erasureTestSetup, error) {
+	diskPaths := make([]string, dataBlocks+parityBlocks)
+	disks := make([]StorageAPI, len(diskPaths))
+	var err error
+	for i := range diskPaths {
+		disks[i], diskPaths[i], err = newPosixTestSetup()
+		if err != nil {
+			return nil, err
+		}
+		err = disks[i].MakeVol("testbucket")
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &erasureTestSetup{dataBlocks, parityBlocks, blockSize, diskPaths, disks}, nil
+}

cmd/errors.go

@@ -0,0 +1,139 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+// Holds the current directory path. Used for trimming path in traceError()
+var rootPath string
+
+// Figure out the rootPath
+func initError() {
+	// Root path is automatically determined from the calling function's source file location.
+	// Catch the calling function's source file path.
+	_, file, _, _ := runtime.Caller(1)
+	// Save the directory alone.
+	rootPath = filepath.Dir(file)
+}
+
+// Represents a stack frame in the stack trace.
+type traceInfo struct {
+	file string // File where error occurred
+	line int    // Line where error occurred
+	name string // Name of the function where error occurred
+}
+
+// Error - error type containing cause and the stack trace.
+type Error struct {
+	e     error       // Holds the cause error
+	trace []traceInfo // stack trace
+	errs  []error     // Useful for XL to hold errors from all disks
+}
+
+// Implement error interface.
+func (e Error) Error() string {
+	return e.e.Error()
+}
+
+// Trace - returns stack trace.
+func (e Error) Trace() []string {
+	var traceArr []string
+	for _, info := range e.trace {
+		traceArr = append(traceArr, fmt.Sprintf("%s:%d:%s",
+			info.file, info.line, info.name))
+	}
+	return traceArr
+}
+
+// NewStorageError - return new Error type.
+func traceError(e error, errs ...error) error {
+	if e == nil {
+		return nil
+	}
+	err := &Error{}
+	err.e = e
+	err.errs = errs
+
+	stack := make([]uintptr, 40)
+	length := runtime.Callers(2, stack)
+	if length > len(stack) {
+		length = len(stack)
+	}
+	stack = stack[:length]
+
+	for _, pc := range stack {
+		pc = pc - 1
+		fn := runtime.FuncForPC(pc)
+		file, line := fn.FileLine(pc)
+		name := fn.Name()
+		if strings.HasSuffix(name, "ServeHTTP") {
+			break
+		}
+		if strings.HasSuffix(name, "runtime.") {
+			break
+		}
+
+		file = strings.TrimPrefix(file, rootPath+string(os.PathSeparator))
+		name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
+		err.trace = append(err.trace, traceInfo{file, line, name})
+	}
+
+	return err
+}
+
+// Returns the underlying cause error.
+func errorCause(err error) error {
+	if e, ok := err.(*Error); ok {
+		err = e.e
+	}
+	return err
+}
+
+// Returns slice of underlying cause error.
+func errorsCause(errs []error) []error {
+	cerrs := make([]error, len(errs))
+	for i, err := range errs {
+		if err == nil {
+			continue
+		}
+		cerrs[i] = errorCause(err)
+	}
+	return cerrs
+}
+
+var baseIgnoredErrs = []error{
+	errDiskNotFound,
+	errFaultyDisk,
+	errFaultyRemoteDisk,
+}
+
+// isErrIgnored returns whether given error is ignored or not.
+func isErrIgnored(err error, ignoredErrs ...error) bool {
+	err = errorCause(err)
+	for _, ignoredErr := range ignoredErrs {
+		if ignoredErr == err {
+			return true
+		}
+	}
+	return false
+}

cmd/event-notifier.go

@@ -0,0 +1,722 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"net"
+	"net/url"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type externalNotifier struct {
+	// Per-bucket notification config. This is updated via
+	// PutBucketNotification API.
+	notificationConfigs map[string]*notificationConfig
+
+	// An external target keeps a connection to an external
+	// service to which events are to be sent. It is a mapping
+	// from an ARN to a log object
+	targets map[string]*logrus.Logger
+
+	rwMutex *sync.RWMutex
+}
+
+type internalNotifier struct {
+	// per-bucket listener configuration. This is updated
+	// when listeners connect or disconnect.
+	listenerConfigs map[string][]listenerConfig
+
+	// An internal target is a peer Minio server, that is
+	// connected to a listening client. Here, targets is a map of
+	// listener ARN to log object.
+	targets map[string]*listenerLogger
+
+	// Connected listeners is a map of listener ARNs to channels
+	// on which the ListenBucket API handler go routine is waiting
+	// for events to send to a client.
+	connectedListeners map[string]chan []NotificationEvent
+
+	rwMutex *sync.RWMutex
+}
+
+// Global event notification configuration. This structure has state
+// about configured external notifications, and run-time configuration
+// for listener notifications.
+type eventNotifier struct {
+
+	// `external` here refers to notification configuration to
+	// send events to supported external systems
+	external externalNotifier
+
+	// `internal` refers to notification configuration for live
+	// listening clients. Events for a client are send from all
+	// servers, internally to a particular server that is
+	// connected to the client.
+	internal internalNotifier
+}
+
+// Represents data to be sent with notification event.
+type eventData struct {
+	Type      EventName
+	Bucket    string
+	ObjInfo   ObjectInfo
+	ReqParams map[string]string
+}
+
+// New notification event constructs a new notification event message from
+// input request metadata which completed successfully.
+func newNotificationEvent(event eventData) NotificationEvent {
+	/// Construct a new object created event.
+	region := serverConfig.GetRegion()
+	tnow := time.Now().UTC()
+	sequencer := fmt.Sprintf("%X", tnow.UnixNano())
+	// Following blocks fills in all the necessary details of s3
+	// event message structure.
+	// http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
+	nEvent := NotificationEvent{
+		EventVersion:      "2.0",
+		EventSource:       "aws:s3",
+		AwsRegion:         region,
+		EventTime:         tnow.Format(timeFormatAMZ),
+		EventName:         event.Type.String(),
+		UserIdentity:      defaultIdentity(),
+		RequestParameters: event.ReqParams,
+		ResponseElements:  map[string]string{},
+		S3: eventMeta{
+			SchemaVersion:   "1.0",
+			ConfigurationID: "Config",
+			Bucket: bucketMeta{
+				Name:          event.Bucket,
+				OwnerIdentity: defaultIdentity(),
+				ARN:           "arn:aws:s3:::" + event.Bucket,
+			},
+		},
+	}
+
+	escapedObj := url.QueryEscape(event.ObjInfo.Name)
+	// For delete object event type, we do not need to set ETag and Size.
+	if event.Type == ObjectRemovedDelete {
+		nEvent.S3.Object = objectMeta{
+			Key:       escapedObj,
+			Sequencer: sequencer,
+		}
+		return nEvent
+	}
+	// For all other events we should set ETag and Size.
+	nEvent.S3.Object = objectMeta{
+		Key:       escapedObj,
+		ETag:      event.ObjInfo.MD5Sum,
+		Size:      event.ObjInfo.Size,
+		Sequencer: sequencer,
+	}
+	// Success.
+	return nEvent
+}
+
+// Fetch the external target. No locking needed here since this map is
+// never written after initial startup.
+func (en eventNotifier) GetExternalTarget(queueARN string) *logrus.Logger {
+	return en.external.targets[queueARN]
+}
+
+func (en eventNotifier) GetInternalTarget(arn string) *listenerLogger {
+	en.internal.rwMutex.RLock()
+	defer en.internal.rwMutex.RUnlock()
+	return en.internal.targets[arn]
+}
+
+// Set a new sns target for an input sns ARN.
+func (en *eventNotifier) AddListenerChan(snsARN string, listenerCh chan []NotificationEvent) error {
+	if listenerCh == nil {
+		return errInvalidArgument
+	}
+	en.internal.rwMutex.Lock()
+	defer en.internal.rwMutex.Unlock()
+	en.internal.connectedListeners[snsARN] = listenerCh
+	return nil
+}
+
+// Remove sns target for an input sns ARN.
+func (en *eventNotifier) RemoveListenerChan(snsARN string) {
+	en.internal.rwMutex.Lock()
+	defer en.internal.rwMutex.Unlock()
+	if en.internal.connectedListeners != nil {
+		delete(en.internal.connectedListeners, snsARN)
+	}
+}
+
+func (en *eventNotifier) SendListenerEvent(arn string, event []NotificationEvent) error {
+	en.internal.rwMutex.Lock()
+	defer en.internal.rwMutex.Unlock()
+
+	ch, ok := en.internal.connectedListeners[arn]
+	if ok {
+		ch <- event
+	}
+	// If the channel is not present we ignore the event.
+	return nil
+}
+
+// Fetch bucket notification config for an input bucket.
+func (en eventNotifier) GetBucketNotificationConfig(bucket string) *notificationConfig {
+	en.external.rwMutex.RLock()
+	defer en.external.rwMutex.RUnlock()
+	return en.external.notificationConfigs[bucket]
+}
+
+func (en *eventNotifier) SetBucketNotificationConfig(bucket string, ncfg *notificationConfig) {
+	en.external.rwMutex.Lock()
+	if ncfg == nil {
+		delete(en.external.notificationConfigs, bucket)
+	} else {
+		en.external.notificationConfigs[bucket] = ncfg
+	}
+	en.external.rwMutex.Unlock()
+}
+
+func (en *eventNotifier) GetBucketListenerConfig(bucket string) []listenerConfig {
+	en.internal.rwMutex.RLock()
+	defer en.internal.rwMutex.RUnlock()
+	return en.internal.listenerConfigs[bucket]
+}
+
+func (en *eventNotifier) SetBucketListenerConfig(bucket string, lcfg []listenerConfig) error {
+	en.internal.rwMutex.Lock()
+	defer en.internal.rwMutex.Unlock()
+	if len(lcfg) == 0 {
+		delete(en.internal.listenerConfigs, bucket)
+	} else {
+		en.internal.listenerConfigs[bucket] = lcfg
+	}
+	for _, elcArr := range en.internal.listenerConfigs {
+		for _, elcElem := range elcArr {
+			currArn := elcElem.TopicConfig.TopicARN
+			logger, err := newListenerLogger(currArn, elcElem.TargetServer)
+			if err != nil {
+				return err
+			}
+			en.internal.targets[currArn] = logger
+		}
+	}
+	return nil
+}
+
+func eventNotifyForBucketNotifications(eventType, objectName, bucketName string, nEvent []NotificationEvent) {
+	nConfig := globalEventNotifier.GetBucketNotificationConfig(bucketName)
+	if nConfig == nil {
+		return
+	}
+	// Validate if the event and object match the queue configs.
+	for _, qConfig := range nConfig.QueueConfigs {
+		eventMatch := eventMatch(eventType, qConfig.Events)
+		ruleMatch := filterRuleMatch(objectName, qConfig.Filter.Key.FilterRules)
+		if eventMatch && ruleMatch {
+			targetLog := globalEventNotifier.GetExternalTarget(qConfig.QueueARN)
+			if targetLog != nil {
+				targetLog.WithFields(logrus.Fields{
+					"Key":       path.Join(bucketName, objectName),
+					"EventType": eventType,
+					"Records":   nEvent,
+				}).Info()
+			}
+		}
+	}
+}
+
+func eventNotifyForBucketListeners(eventType, objectName, bucketName string,
+	nEvent []NotificationEvent) {
+	lCfgs := globalEventNotifier.GetBucketListenerConfig(bucketName)
+	if lCfgs == nil {
+		return
+	}
+	// Validate if the event and object match listener configs
+	for _, lcfg := range lCfgs {
+		ruleMatch := filterRuleMatch(objectName, lcfg.TopicConfig.Filter.Key.FilterRules)
+		eventMatch := eventMatch(eventType, lcfg.TopicConfig.Events)
+		if eventMatch && ruleMatch {
+			targetLog := globalEventNotifier.GetInternalTarget(
+				lcfg.TopicConfig.TopicARN)
+			if targetLog != nil && targetLog.log != nil {
+				targetLog.log.WithFields(logrus.Fields{
+					"Key":       path.Join(bucketName, objectName),
+					"EventType": eventType,
+					"Records":   nEvent,
+				}).Info()
+			}
+		}
+	}
+
+}
+
+// eventNotify notifies an event to relevant targets based on their
+// bucket configuration (notifications and listeners).
+func eventNotify(event eventData) {
+	// Notifies a new event.
+	// List of events reported through this function are
+	//  - s3:ObjectCreated:Put
+	//  - s3:ObjectCreated:Post
+	//  - s3:ObjectCreated:Copy
+	//  - s3:ObjectCreated:CompleteMultipartUpload
+	//  - s3:ObjectRemoved:Delete
+
+	// Event type.
+	eventType := event.Type.String()
+
+	// Object name.
+	objectName := event.ObjInfo.Name
+
+	// Save the notification event to be sent.
+	notificationEvent := []NotificationEvent{newNotificationEvent(event)}
+
+	// Notify external targets.
+	eventNotifyForBucketNotifications(eventType, objectName, event.Bucket, notificationEvent)
+
+	// Notify internal targets.
+	eventNotifyForBucketListeners(eventType, objectName, event.Bucket, notificationEvent)
+}
+
+// loads notification config if any for a given bucket, returns
+// structured notification config.
+func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationConfig, error) {
+	// Construct the notification config path.
+	ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
+
+	// Acquire a write lock on notification config before modifying.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
+	objLock.RLock()
+	defer objLock.RUnlock()
+
+	objInfo, err := objAPI.GetObjectInfo(minioMetaBucket, ncPath)
+	if err != nil {
+		// 'notification.xml' not found return
+		// 'errNoSuchNotifications'.  This is default when no
+		// bucket notifications are found on the bucket.
+		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
+			return nil, errNoSuchNotifications
+		}
+		errorIf(err, "Unable to load bucket-notification for bucket %s", bucket)
+		// Returns error for other errors.
+		return nil, err
+	}
+	var buffer bytes.Buffer
+	err = objAPI.GetObject(minioMetaBucket, ncPath, 0, objInfo.Size, &buffer)
+	if err != nil {
+		// 'notification.xml' not found return
+		// 'errNoSuchNotifications'.  This is default when no
+		// bucket notifications are found on the bucket.
+		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
+			return nil, errNoSuchNotifications
+		}
+		errorIf(err, "Unable to load bucket-notification for bucket %s", bucket)
+		// Returns error for other errors.
+		return nil, err
+	}
+
+	// Unmarshal notification bytes.
+	notificationConfigBytes := buffer.Bytes()
+	notificationCfg := &notificationConfig{}
+	if err = xml.Unmarshal(notificationConfigBytes, &notificationCfg); err != nil {
+		return nil, err
+	}
+
+	// Return success.
+	return notificationCfg, nil
+}
+
+// loads notification config if any for a given bucket, returns
+// structured notification config.
+func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, error) {
+	// in single node mode, there are no peers, so in this case
+	// there is no configuration to load, as any previously
+	// connected listen clients have been disconnected
+	if !globalIsDistXL {
+		return nil, nil
+	}
+
+	// Construct the notification config path.
+	lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
+
+	// Acquire a write lock on notification config before modifying.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
+	objLock.RLock()
+	defer objLock.RUnlock()
+
+	objInfo, err := objAPI.GetObjectInfo(minioMetaBucket, lcPath)
+	if err != nil {
+		// 'listener.json' not found return
+		// 'errNoSuchNotifications'.  This is default when no
+		// bucket notifications are found on the bucket.
+		if isErrObjectNotFound(err) {
+			return nil, errNoSuchNotifications
+		}
+		errorIf(err, "Unable to load bucket-listeners for bucket %s", bucket)
+		// Returns error for other errors.
+		return nil, err
+	}
+	var buffer bytes.Buffer
+	err = objAPI.GetObject(minioMetaBucket, lcPath, 0, objInfo.Size, &buffer)
+	if err != nil {
+		// 'notification.xml' not found return
+		// 'errNoSuchNotifications'.  This is default when no
+		// bucket listeners are found on the bucket.
+		if isErrObjectNotFound(err) {
+			return nil, errNoSuchNotifications
+		}
+		errorIf(err, "Unable to load bucket-listeners for bucket %s", bucket)
+		// Returns error for other errors.
+		return nil, err
+	}
+
+	// Unmarshal notification bytes.
+	var lCfg []listenerConfig
+	lConfigBytes := buffer.Bytes()
+	if err = json.Unmarshal(lConfigBytes, &lCfg); err != nil {
+		errorIf(err, "Unable to unmarshal listener config from JSON.")
+		return nil, err
+	}
+
+	// Return success.
+	return lCfg, nil
+}
+
+func persistNotificationConfig(bucket string, ncfg *notificationConfig, obj ObjectLayer) error {
+	// marshal to xml
+	buf, err := xml.Marshal(ncfg)
+	if err != nil {
+		errorIf(err, "Unable to marshal notification configuration into XML")
+		return err
+	}
+
+	// build path
+	ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
+	// Acquire a write lock on notification config before modifying.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
+	objLock.Lock()
+	defer objLock.Unlock()
+
+	// write object to path
+	sha256Sum := getSHA256Hash(buf)
+	_, err = obj.PutObject(minioMetaBucket, ncPath, int64(len(buf)), bytes.NewReader(buf), nil, sha256Sum)
+	if err != nil {
+		errorIf(err, "Unable to write bucket notification configuration.")
+		return err
+	}
+	return nil
+}
+
+// Persists validated listener config to object layer.
+func persistListenerConfig(bucket string, lcfg []listenerConfig, obj ObjectLayer) error {
+	buf, err := json.Marshal(lcfg)
+	if err != nil {
+		errorIf(err, "Unable to marshal listener config to JSON.")
+		return err
+	}
+
+	// build path
+	lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
+	// Acquire a write lock on notification config before modifying.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
+	objLock.Lock()
+	defer objLock.Unlock()
+
+	// write object to path
+	sha256Sum := getSHA256Hash(buf)
+	_, err = obj.PutObject(minioMetaBucket, lcPath, int64(len(buf)), bytes.NewReader(buf), nil, sha256Sum)
+	if err != nil {
+		errorIf(err, "Unable to write bucket listener configuration to object layer.")
+	}
+	return err
+}
+
+// Removes notification.xml for a given bucket, only used during DeleteBucket.
+func removeNotificationConfig(bucket string, objAPI ObjectLayer) error {
+	// Verify bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+
+	ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
+
+	// Acquire a write lock on notification config before modifying.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
+	objLock.Lock()
+	err := objAPI.DeleteObject(minioMetaBucket, ncPath)
+	objLock.Unlock()
+	return err
+}
+
+// Remove listener configuration from storage layer. Used when a bucket is deleted.
+func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
+	// make the path
+	lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
+
+	// Acquire a write lock on notification config before modifying.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
+	objLock.Lock()
+	err := objAPI.DeleteObject(minioMetaBucket, lcPath)
+	objLock.Unlock()
+	return err
+}
+
+// Loads both notification and listener config.
+func loadNotificationAndListenerConfig(bucketName string, objAPI ObjectLayer) (nCfg *notificationConfig, lCfg []listenerConfig, err error) {
+	// Loads notification config if any.
+	nCfg, err = loadNotificationConfig(bucketName, objAPI)
+	if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+		return nil, nil, err
+	}
+
+	// Loads listener config if any.
+	lCfg, err = loadListenerConfig(bucketName, objAPI)
+	if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+		return nil, nil, err
+	}
+	return nCfg, lCfg, nil
+}
+
+// loads all bucket notifications if present.
+func loadAllBucketNotifications(objAPI ObjectLayer) (map[string]*notificationConfig, map[string][]listenerConfig, error) {
+	// List buckets to proceed loading all notification configuration.
+	buckets, err := objAPI.ListBuckets()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	nConfigs := make(map[string]*notificationConfig)
+	lConfigs := make(map[string][]listenerConfig)
+
+	// Loads all bucket notifications.
+	for _, bucket := range buckets {
+		// Load persistent notification and listener configurations
+		// a given bucket name.
+		nConfigs[bucket.Name], lConfigs[bucket.Name], err = loadNotificationAndListenerConfig(bucket.Name, objAPI)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	// Success.
+	return nConfigs, lConfigs, nil
+}
+
+// Loads all queue targets, initializes each queueARNs depending on their config.
+// Each instance of queueARN registers its own logrus to communicate with the
+// queue service. QueueARN once initialized is not initialized again for the
+// same queueARN, instead previous connection is used.
+func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
+	queueTargets := make(map[string]*logrus.Logger)
+	// Load all amqp targets, initialize their respective loggers.
+	for accountID, amqpN := range serverConfig.GetAMQP() {
+		if !amqpN.Enable {
+			continue
+		}
+		// Construct the queue ARN for AMQP.
+		queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueTypeAMQP
+		// Queue target if already initialized we move to the next ARN.
+		_, ok := queueTargets[queueARN]
+		if ok {
+			continue
+		}
+		// Using accountID we can now initialize a new AMQP logrus instance.
+		amqpLog, err := newAMQPNotify(accountID)
+		if err != nil {
+			// Encapsulate network error to be more informative.
+			if _, ok := err.(net.Error); ok {
+				return nil, &net.OpError{
+					Op:  "Connecting to " + queueARN,
+					Net: "tcp",
+					Err: err,
+				}
+			}
+			return nil, err
+		}
+		queueTargets[queueARN] = amqpLog
+	}
+	// Load all nats targets, initialize their respective loggers.
+	for accountID, natsN := range serverConfig.GetNATS() {
+		if !natsN.Enable {
+			continue
+		}
+		// Construct the queue ARN for NATS.
+		queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueTypeNATS
+		// Queue target if already initialized we move to the next ARN.
+		_, ok := queueTargets[queueARN]
+		if ok {
+			continue
+		}
+		// Using accountID we can now initialize a new NATS logrus instance.
+		natsLog, err := newNATSNotify(accountID)
+		if err != nil {
+			// Encapsulate network error to be more informative.
+			if _, ok := err.(net.Error); ok {
+				return nil, &net.OpError{
+					Op:  "Connecting to " + queueARN,
+					Net: "tcp",
+					Err: err,
+				}
+			}
+			return nil, err
+		}
+		queueTargets[queueARN] = natsLog
+	}
+
+	// Load redis targets, initialize their respective loggers.
+	for accountID, redisN := range serverConfig.GetRedis() {
+		if !redisN.Enable {
+			continue
+		}
+		// Construct the queue ARN for Redis.
+		queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueTypeRedis
+		// Queue target if already initialized we move to the next ARN.
+		_, ok := queueTargets[queueARN]
+		if ok {
+			continue
+		}
+		// Using accountID we can now initialize a new Redis logrus instance.
+		redisLog, err := newRedisNotify(accountID)
+		if err != nil {
+			// Encapsulate network error to be more informative.
+			if _, ok := err.(net.Error); ok {
+				return nil, &net.OpError{
+					Op:  "Connecting to " + queueARN,
+					Net: "tcp",
+					Err: err,
+				}
+			}
+			return nil, err
+		}
+		queueTargets[queueARN] = redisLog
+	}
+	// Load elastic targets, initialize their respective loggers.
+	for accountID, elasticN := range serverConfig.GetElasticSearch() {
+		if !elasticN.Enable {
+			continue
+		}
+		// Construct the queue ARN for Elastic.
+		queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueTypeElastic
+		_, ok := queueTargets[queueARN]
+		if ok {
+			continue
+		}
+		// Using accountID we can now initialize a new ElasticSearch logrus instance.
+		elasticLog, err := newElasticNotify(accountID)
+		if err != nil {
+			// Encapsulate network error to be more informative.
+			if _, ok := err.(net.Error); ok {
+				return nil, &net.OpError{
+					Op: "Connecting to " + queueARN, Net: "tcp",
+					Err: err,
+				}
+			}
+			return nil, err
+		}
+		queueTargets[queueARN] = elasticLog
+	}
+	// Load PostgreSQL targets, initialize their respective loggers.
+	for accountID, pgN := range serverConfig.GetPostgreSQL() {
+		if !pgN.Enable {
+			continue
+		}
+		// Construct the queue ARN for Postgres.
+		queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueTypePostgreSQL
+		_, ok := queueTargets[queueARN]
+		if ok {
+			continue
+		}
+		// Using accountID initialize a new Postgresql logrus instance.
+		pgLog, err := newPostgreSQLNotify(accountID)
+		if err != nil {
+			// Encapsulate network error to be more informative.
+			if _, ok := err.(net.Error); ok {
+				return nil, &net.OpError{
+					Op: "Connecting to " + queueARN, Net: "tcp",
+					Err: err,
+				}
+			}
+			return nil, err
+		}
+		queueTargets[queueARN] = pgLog
+	}
+
+	// Successfully initialized queue targets.
+	return queueTargets, nil
+}
+
+// Global instance of event notification queue.
+var globalEventNotifier *eventNotifier
+
+// Initialize event notifier.
+func initEventNotifier(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+
+	// Read all saved bucket notifications.
+	nConfigs, lConfigs, err := loadAllBucketNotifications(objAPI)
+	if err != nil {
+		errorIf(err, "Error loading bucket notifications - %v", err)
+		return err
+	}
+
+	// Initializes all queue targets.
+	queueTargets, err := loadAllQueueTargets()
+	if err != nil {
+		return err
+	}
+
+	// Initialize internal listener targets
+	listenTargets := make(map[string]*listenerLogger)
+	for _, listeners := range lConfigs {
+		for _, listener := range listeners {
+			ln, err := newListenerLogger(
+				listener.TopicConfig.TopicARN,
+				listener.TargetServer,
+			)
+			if err != nil {
+				errorIf(err, "Unable to initialize listener target logger.")
+				//TODO: improve error
+				return fmt.Errorf("Error initializing listner target logger - %v", err)
+			}
+			listenTargets[listener.TopicConfig.TopicARN] = ln
+		}
+	}
+
+	// Initialize event notifier queue.
+	globalEventNotifier = &eventNotifier{
+		external: externalNotifier{
+			notificationConfigs: nConfigs,
+			targets:             queueTargets,
+			rwMutex:             &sync.RWMutex{},
+		},
+		internal: internalNotifier{
+			rwMutex:            &sync.RWMutex{},
+			targets:            listenTargets,
+			listenerConfigs:    lConfigs,
+			connectedListeners: make(map[string]chan []NotificationEvent),
+		},
+	}
+
+	return nil
+}

cmd/event-notifier_test.go

@@ -0,0 +1,513 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+	"testing"
+	"time"
+)
+
+// Test InitEventNotifier with faulty disks
+func TestInitEventNotifierFaultyDisks(t *testing.T) {
+	// Prepare for tests
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	disks, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal("Unable to create directories for FS backend. ", err)
+	}
+	defer removeAll(disks[0])
+	endpoints, err := parseStorageEndpoints(disks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	obj, _, err := initObjectLayer(endpoints)
+	if err != nil {
+		t.Fatal("Unable to initialize FS backend.", err)
+	}
+
+	bucketName := "bucket"
+	if err := obj.MakeBucket(bucketName); err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	fs := obj.(fsObjects)
+	fsstorage := fs.storage.(*retryStorage)
+
+	listenARN := "arn:minio:sns:us-east-1:1:listen"
+	queueARN := "arn:minio:sqs:us-east-1:1:redis"
+
+	// Write a notification.xml in the disk
+	notificationXML := "<NotificationConfiguration>"
+	notificationXML += "<TopicConfiguration><Event>s3:ObjectRemoved:*</Event><Event>s3:ObjectRemoved:*</Event><Topic>" + listenARN + "</Topic></TopicConfiguration>"
+	notificationXML += "<QueueConfiguration><Event>s3:ObjectRemoved:*</Event><Event>s3:ObjectRemoved:*</Event><Queue>" + queueARN + "</Queue></QueueConfiguration>"
+	notificationXML += "</NotificationConfiguration>"
+	if err := fsstorage.AppendFile(minioMetaBucket, bucketConfigPrefix+"/"+bucketName+"/"+bucketNotificationConfig, []byte(notificationXML)); err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	// Test initEventNotifier() with faulty disks
+	for i := 1; i <= 5; i++ {
+		fs.storage = newNaughtyDisk(fsstorage, map[int]error{i: errFaultyDisk}, nil)
+		if err := initEventNotifier(fs); errorCause(err) != errFaultyDisk {
+			t.Fatal("Unexpected error:", err)
+		}
+	}
+}
+
+// InitEventNotifierWithAMQP - tests InitEventNotifier when AMQP is not prepared
+func TestInitEventNotifierWithAMQP(t *testing.T) {
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	disks, err := getRandomDisks(1)
+	defer removeAll(disks[0])
+	if err != nil {
+		t.Fatal("Unable to create directories for FS backend. ", err)
+	}
+	endpoints, err := parseStorageEndpoints(disks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fs, _, err := initObjectLayer(endpoints)
+	if err != nil {
+		t.Fatal("Unable to initialize FS backend.", err)
+	}
+
+	serverConfig.SetAMQPNotifyByID("1", amqpNotify{Enable: true})
+	if err := initEventNotifier(fs); err == nil {
+		t.Fatal("AMQP config didn't fail.")
+	}
+}
+
+// InitEventNotifierWithElasticSearch - test InitEventNotifier when ElasticSearch is not ready
+func TestInitEventNotifierWithElasticSearch(t *testing.T) {
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	disks, err := getRandomDisks(1)
+	defer removeAll(disks[0])
+	if err != nil {
+		t.Fatal("Unable to create directories for FS backend. ", err)
+	}
+	endpoints, err := parseStorageEndpoints(disks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fs, _, err := initObjectLayer(endpoints)
+	if err != nil {
+		t.Fatal("Unable to initialize FS backend.", err)
+	}
+
+	serverConfig.SetElasticSearchNotifyByID("1", elasticSearchNotify{Enable: true})
+	if err := initEventNotifier(fs); err == nil {
+		t.Fatal("ElasticSearch config didn't fail.")
+	}
+}
+
+// InitEventNotifierWithRedis - test InitEventNotifier when Redis is not ready
+func TestInitEventNotifierWithRedis(t *testing.T) {
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer removeAll(rootPath)
+
+	disks, err := getRandomDisks(1)
+	defer removeAll(disks[0])
+	if err != nil {
+		t.Fatal("Unable to create directories for FS backend. ", err)
+	}
+	endpoints, err := parseStorageEndpoints(disks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fs, _, err := initObjectLayer(endpoints)
+	if err != nil {
+		t.Fatal("Unable to initialize FS backend.", err)
+	}
+
+	serverConfig.SetRedisNotifyByID("1", redisNotify{Enable: true})
+	if err := initEventNotifier(fs); err == nil {
+		t.Fatal("Redis config didn't fail.")
+	}
+}
+
+type TestPeerRPCServerData struct {
+	serverType string
+	testServer TestServer
+}
+
+func (s *TestPeerRPCServerData) Setup(t *testing.T) {
+	s.testServer = StartTestPeersRPCServer(t, s.serverType)
+
+	// setup port and minio addr
+	host, port, err := net.SplitHostPort(s.testServer.Server.Listener.Addr().String())
+	if err != nil {
+		t.Fatalf("Initialisation error: %v", err)
+	}
+	globalMinioHost = host
+	globalMinioPort = port
+	globalMinioAddr = getLocalAddress(
+		s.testServer.SrvCmdCfg,
+	)
+
+	// initialize the peer client(s)
+	initGlobalS3Peers(s.testServer.Disks)
+}
+
+func (s *TestPeerRPCServerData) TearDown() {
+	s.testServer.Stop()
+	_ = removeAll(s.testServer.Root)
+	for _, d := range s.testServer.Disks {
+		_ = removeAll(d.Path)
+	}
+}
+
+func TestSetNGetBucketNotification(t *testing.T) {
+	s := TestPeerRPCServerData{serverType: "XL"}
+
+	// setup and teardown
+	s.Setup(t)
+	defer s.TearDown()
+
+	bucketName := getRandomBucketName()
+
+	obj := s.testServer.Obj
+	if err := initEventNotifier(obj); err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	globalEventNotifier.SetBucketNotificationConfig(bucketName, &notificationConfig{})
+	nConfig := globalEventNotifier.GetBucketNotificationConfig(bucketName)
+	if nConfig == nil {
+		t.Errorf("Notification expected to be set, but notification not set.")
+	}
+
+	if !reflect.DeepEqual(nConfig, &notificationConfig{}) {
+		t.Errorf("Mismatching notification configs.")
+	}
+}
+
+func TestInitEventNotifier(t *testing.T) {
+	s := TestPeerRPCServerData{serverType: "XL"}
+
+	// setup and teardown
+	s.Setup(t)
+	defer s.TearDown()
+
+	// test if empty object layer arg. returns expected error.
+	if err := initEventNotifier(nil); err == nil || err != errInvalidArgument {
+		t.Fatalf("initEventNotifier returned unexpected error value - %v", err)
+	}
+
+	obj := s.testServer.Obj
+	bucketName := getRandomBucketName()
+	// declare sample configs
+	filterRules := []filterRule{
+		{
+			Name:  "prefix",
+			Value: "minio",
+		},
+		{
+			Name:  "suffix",
+			Value: "*.jpg",
+		},
+	}
+	sampleSvcCfg := ServiceConfig{
+		[]string{"s3:ObjectRemoved:*", "s3:ObjectCreated:*"},
+		filterStruct{
+			keyFilter{filterRules},
+		},
+		"1",
+	}
+	sampleNotifCfg := notificationConfig{
+		QueueConfigs: []queueConfig{
+			{
+				ServiceConfig: sampleSvcCfg,
+				QueueARN:      "testqARN",
+			},
+		},
+	}
+	sampleListenCfg := []listenerConfig{
+		{
+			TopicConfig: topicConfig{ServiceConfig: sampleSvcCfg,
+				TopicARN: "testlARN"},
+			TargetServer: globalMinioAddr,
+		},
+	}
+
+	// create bucket
+	if err := obj.MakeBucket(bucketName); err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	// bucket is created, now writing should not give errors.
+	if err := persistNotificationConfig(bucketName, &sampleNotifCfg, obj); err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	if err := persistListenerConfig(bucketName, sampleListenCfg, obj); err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	// needed to load listener config from disk for testing (in
+	// single peer mode, the listener config is ingored, but here
+	// we want to test the loading from disk too.)
+	globalIsDistXL = true
+
+	// test event notifier init
+	if err := initEventNotifier(obj); err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	// fetch bucket configs and verify
+	ncfg := globalEventNotifier.GetBucketNotificationConfig(bucketName)
+	if ncfg == nil {
+		t.Error("Bucket notification was not present for ", bucketName)
+	}
+	if len(ncfg.QueueConfigs) != 1 || ncfg.QueueConfigs[0].QueueARN != "testqARN" {
+		t.Error("Unexpected bucket notification found - ", *ncfg)
+	}
+	if globalEventNotifier.GetExternalTarget("testqARN") != nil {
+		t.Error("A logger was not expected to be found as it was not enabled in the config.")
+	}
+
+	lcfg := globalEventNotifier.GetBucketListenerConfig(bucketName)
+	if lcfg == nil {
+		t.Error("Bucket listener was not present for ", bucketName)
+	}
+	if len(lcfg) != 1 || lcfg[0].TargetServer != globalMinioAddr || lcfg[0].TopicConfig.TopicARN != "testlARN" {
+		t.Error("Unexpected listener config found - ", lcfg[0])
+	}
+	if globalEventNotifier.GetInternalTarget("testlARN") == nil {
+		t.Error("A listen logger was not found.")
+	}
+}
+
+func TestListenBucketNotification(t *testing.T) {
+	s := TestPeerRPCServerData{serverType: "XL"}
+	// setup and teardown
+	s.Setup(t)
+	defer s.TearDown()
+
+	// test initialisation
+	obj := s.testServer.Obj
+
+	bucketName := "bucket"
+	objectName := "object"
+
+	// Create the bucket to listen on
+	if err := obj.MakeBucket(bucketName); err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	listenARN := fmt.Sprintf("%s:%s:1:%s-%s",
+		minioTopic,
+		serverConfig.GetRegion(),
+		snsTypeMinio,
+		s.testServer.Server.Listener.Addr(),
+	)
+	lcfg := listenerConfig{
+		TopicConfig: topicConfig{
+			ServiceConfig{
+				[]string{"s3:ObjectRemoved:*", "s3:ObjectCreated:*"},
+				filterStruct{},
+				"0",
+			},
+			listenARN,
+		},
+		TargetServer: globalMinioAddr,
+	}
+
+	// write listener config to storage layer
+	lcfgs := []listenerConfig{lcfg}
+	if err := persistListenerConfig(bucketName, lcfgs, obj); err != nil {
+		t.Fatalf("Test Setup error: %v", err)
+	}
+
+	// needed to load listener config from disk for testing (in
+	// single peer mode, the listener config is ingored, but here
+	// we want to test the loading from disk too.)
+	globalIsDistXL = true
+
+	// Init event notifier
+	if err := initEventNotifier(obj); err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	// Check if the config is loaded
+	listenerCfg := globalEventNotifier.GetBucketListenerConfig(bucketName)
+	if listenerCfg == nil {
+		t.Fatal("Cannot load bucket listener config")
+	}
+	if len(listenerCfg) != 1 {
+		t.Fatal("Listener config is not correctly loaded. Exactly one listener config is expected")
+	}
+
+	// Check if topic ARN is correct
+	if listenerCfg[0].TopicConfig.TopicARN != listenARN {
+		t.Fatal("Configured topic ARN is incorrect.")
+	}
+
+	// Create a new notification event channel.
+	nEventCh := make(chan []NotificationEvent)
+	// Close the listener channel.
+	defer close(nEventCh)
+	// Add events channel for listener.
+	if err := globalEventNotifier.AddListenerChan(listenARN, nEventCh); err != nil {
+		t.Fatalf("Test Setup error: %v", err)
+	}
+	// Remove listen channel after the writer has closed or the
+	// client disconnected.
+	defer globalEventNotifier.RemoveListenerChan(listenARN)
+
+	// Fire an event notification
+	go eventNotify(eventData{
+		Type:   ObjectRemovedDelete,
+		Bucket: bucketName,
+		ObjInfo: ObjectInfo{
+			Bucket: bucketName,
+			Name:   objectName,
+		},
+		ReqParams: map[string]string{
+			"sourceIPAddress": "localhost:1337",
+		},
+	})
+
+	// Wait for the event notification here, if nothing is received within 30 seconds,
+	// test error will be fired
+	select {
+	case n := <-nEventCh:
+		// Check that received event
+		if len(n) == 0 {
+			t.Fatal("Unexpected error occurred")
+		}
+		if n[0].S3.Object.Key != objectName {
+			t.Fatalf("Received wrong object name in notification, expected %s, received %s", n[0].S3.Object.Key, objectName)
+		}
+		break
+	case <-time.After(3 * time.Second):
+		break
+	}
+
+}
+
+func TestAddRemoveBucketListenerConfig(t *testing.T) {
+	s := TestPeerRPCServerData{serverType: "XL"}
+
+	// setup and teardown
+	s.Setup(t)
+	defer s.TearDown()
+
+	// test code
+	obj := s.testServer.Obj
+	if err := initEventNotifier(obj); err != nil {
+		t.Fatalf("Failed to initialize event notifier: %v", err)
+	}
+
+	// Make a bucket to store topicConfigs.
+	randBucket := getRandomBucketName()
+	if err := obj.MakeBucket(randBucket); err != nil {
+		t.Fatalf("Failed to make bucket %s", randBucket)
+	}
+
+	// Add a topicConfig to an empty notificationConfig.
+	accountID := fmt.Sprintf("%d", time.Now().UTC().UnixNano())
+	accountARN := fmt.Sprintf(
+		"arn:minio:sqs:%s:%s:listen-%s",
+		serverConfig.GetRegion(),
+		accountID,
+		globalMinioAddr,
+	)
+
+	// Make topic configuration
+	filterRules := []filterRule{
+		{
+			Name:  "prefix",
+			Value: "minio",
+		},
+		{
+			Name:  "suffix",
+			Value: "*.jpg",
+		},
+	}
+	sampleTopicCfg := topicConfig{
+		TopicARN: accountARN,
+		ServiceConfig: ServiceConfig{
+			[]string{"s3:ObjectRemoved:*", "s3:ObjectCreated:*"},
+			filterStruct{
+				keyFilter{filterRules},
+			},
+			"sns-" + accountID,
+		},
+	}
+	sampleListenerCfg := &listenerConfig{
+		TopicConfig:  sampleTopicCfg,
+		TargetServer: globalMinioAddr,
+	}
+	testCases := []struct {
+		lCfg        *listenerConfig
+		expectedErr error
+	}{
+		{sampleListenerCfg, nil},
+		{nil, errInvalidArgument},
+	}
+
+	for i, test := range testCases {
+		err := AddBucketListenerConfig(randBucket, test.lCfg, obj)
+		if err != test.expectedErr {
+			t.Errorf(
+				"Test %d: Failed with error %v, expected to fail with %v",
+				i+1, err, test.expectedErr,
+			)
+		}
+	}
+
+	// test remove listener actually removes a listener
+	RemoveBucketListenerConfig(randBucket, sampleListenerCfg, obj)
+	// since it does not return errors we fetch the config and
+	// check
+	lcSlice := globalEventNotifier.GetBucketListenerConfig(randBucket)
+	if len(lcSlice) != 0 {
+		t.Errorf("Remove Listener Config Test: did not remove listener config - %v",
+			lcSlice)
+	}
+}

cmd/fallocate.go

@@ -0,0 +1,25 @@
+// +build !linux
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// Fallocate is not POSIX and not supported under Windows
+// Always return successful
+func Fallocate(fd int, offset int64, len int64) error {
+	return nil
+}

cmd/fallocate_linux.go

@@ -0,0 +1,31 @@
+// +build linux
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "syscall"
+
+// Fallocate uses the linux Fallocate syscall, which helps us to be
+// sure that subsequent writes on a file just created will not fail,
+// in addition, file allocation will be contigous on the disk
+func Fallocate(fd int, offset int64, len int64) error {
+	return syscall.Fallocate(fd,
+		1, // FALLOC_FL_KEEP_SIZE
+		offset,
+		len)
+}

cmd/format-config-v1.go

@@ -0,0 +1,922 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+)
+
+// fsFormat - structure holding 'fs' format.
+type fsFormat struct {
+	Version string `json:"version"`
+}
+
+// xlFormat - structure holding 'xl' format.
+type xlFormat struct {
+	Version string `json:"version"` // Version of 'xl' format.
+	Disk    string `json:"disk"`    // Disk field carries assigned disk uuid.
+	// JBOD field carries the input disk order generated the first
+	// time when fresh disks were supplied.
+	JBOD []string `json:"jbod"`
+}
+
+// formatConfigV1 - structure holds format config version '1'.
+type formatConfigV1 struct {
+	Version string `json:"version"` // Version of the format config.
+	// Format indicates the backend format type, supports two values 'xl' and 'fs'.
+	Format string    `json:"format"`
+	FS     *fsFormat `json:"fs,omitempty"` // FS field holds fs format.
+	XL     *xlFormat `json:"xl,omitempty"` // XL field holds xl format.
+}
+
+/*
+
+All disks online
+-----------------
+- All Unformatted - format all and return success.
+- Some Unformatted - format all and return success.
+- Any JBOD inconsistent - return failure // Requires deep inspection, phase2.
+- Some are corrupt (missing format.json) - return failure  // Requires deep inspection, phase2.
+- Any unrecognized disks - return failure
+
+Some disks are offline and we have quorum.
+-----------------
+- Some unformatted - no heal, return success.
+- Any JBOD inconsistent - return failure // Requires deep inspection, phase2.
+- Some are corrupt (missing format.json) - return failure  // Requires deep inspection, phase2.
+- Any unrecognized disks - return failure
+
+No read quorum
+-----------------
+failure for all cases.
+
+// Pseudo code for managing `format.json`.
+
+// Generic checks.
+if (no quorum) return error
+if (any disk is corrupt) return error // phase2
+if (jbod inconsistent) return error // phase2
+if (disks not recognized) // Always error.
+
+// Specific checks.
+if (all disks online)
+  if (all disks return format.json)
+     if (jbod consistent)
+        if (all disks recognized)
+          return
+  else
+     if (all disks return format.json not found)
+        (initialize format)
+        return
+     else (some disks return format.json not found)
+        (heal format)
+        return
+     fi
+   fi
+else // No healing at this point forward, some disks are offline or dead.
+   if (some disks return format.json not found)
+      if (with force)
+         // Offline disks are marked as dead.
+         (heal format) // Offline disks should be marked as dead.
+         return success
+      else (without force)
+         // --force is necessary to heal few drives, because some drives
+         // are offline. Offline disks will be marked as dead.
+         return error
+      fi
+fi
+*/
+
+// error returned when some disks are found to be unformatted.
+var errSomeDiskUnformatted = errors.New("some disks are found to be unformatted")
+
+// error returned when some disks are offline.
+var errSomeDiskOffline = errors.New("some disks are offline")
+
+// errDiskOrderMismatch - returned when disk UUID is not in consistent JBOD order.
+var errDiskOrderMismatch = errors.New("disk order mismatch")
+
+// Returns error slice into understandable errors.
+func reduceFormatErrs(errs []error, diskCount int) (err error) {
+	var errUnformattedDiskCount = 0
+	var errDiskNotFoundCount = 0
+	var errCorruptedFormatCount = 0
+	for _, dErr := range errs {
+		if dErr == errUnformattedDisk {
+			errUnformattedDiskCount++
+		} else if dErr == errDiskNotFound {
+			errDiskNotFoundCount++
+		} else if dErr == errCorruptedFormat {
+			errCorruptedFormatCount++
+		}
+	}
+	if errCorruptedFormatCount > 0 {
+		return errCorruptedFormat
+	}
+	// Unformatted disks found, we need to figure out if any disks are offline.
+	if errUnformattedDiskCount > 0 {
+		// Returns errUnformattedDisk if all disks report unFormattedDisk.
+		if errUnformattedDiskCount < diskCount {
+			if errDiskNotFoundCount > 0 {
+				// Only some disks are fresh but some disks are offline as well.
+				return errSomeDiskOffline
+			}
+			// Some disks are fresh disks an unformatted, not disks are offline.
+			return errSomeDiskUnformatted
+		}
+		// All disks returned unformatted, all disks must be fresh.
+		return errUnformattedDisk
+	}
+	// No unformatted disks found no need to handle disk not found case, return success here.
+	return nil
+}
+
+// creates format.json, the FS format info in minioMetaBucket.
+func initFormatFS(storageDisk StorageAPI) error {
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolume([]StorageAPI{storageDisk}); err != nil {
+		return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
+	}
+	return saveFSFormatData(storageDisk, newFSFormatV1())
+}
+
+// loads format.json from minioMetaBucket if it exists.
+func loadFormatFS(storageDisk StorageAPI) (format *formatConfigV1, err error) {
+	return loadFormat(storageDisk)
+}
+
+// loadAllFormats - load all format config from all input disks in parallel.
+func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
+	// Initialize sync waitgroup.
+	var wg = &sync.WaitGroup{}
+
+	// Initialize list of errors.
+	var sErrs = make([]error, len(bootstrapDisks))
+
+	// Initialize format configs.
+	var formatConfigs = make([]*formatConfigV1, len(bootstrapDisks))
+
+	// Make a volume entry on all underlying storage disks.
+	for index, disk := range bootstrapDisks {
+		if disk == nil {
+			sErrs[index] = errDiskNotFound
+			continue
+		}
+		wg.Add(1)
+		// Make a volume inside a go-routine.
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			formatConfig, lErr := loadFormat(disk)
+			if lErr != nil {
+				sErrs[index] = lErr
+				return
+			}
+			formatConfigs[index] = formatConfig
+		}(index, disk)
+	}
+
+	// Wait for all make vol to finish.
+	wg.Wait()
+
+	for _, err := range sErrs {
+		if err != nil {
+			// Return all formats and errors.
+			return formatConfigs, sErrs
+		}
+	}
+	// Return all formats and nil
+	return formatConfigs, sErrs
+}
+
+// genericFormatCheckFS - validates format config and returns an error if any.
+func genericFormatCheckFS(formatConfig *formatConfigV1, sErr error) (err error) {
+	if sErr != nil {
+		return sErr
+	}
+	// Successfully read, validate if FS.
+	if !isFSFormat(formatConfig) {
+		return errFSDiskFormat
+	}
+	return nil
+}
+
+// genericFormatCheckXL - validates and returns error.
+// if (no quorum) return error
+// if (any disk is corrupt) return error // phase2
+// if (jbod inconsistent) return error // phase2
+// if (disks not recognized) // Always error.
+func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err error) {
+	// Calculate the errors.
+	var (
+		errCorruptFormatCount = 0
+		errCount              = 0
+	)
+
+	// Through all errors calculate the actual errors.
+	for _, lErr := range sErrs {
+		if lErr == nil {
+			continue
+		}
+		// These errors are good conditions, means disk is online.
+		if lErr == errUnformattedDisk || lErr == errVolumeNotFound {
+			continue
+		}
+		if lErr == errCorruptedFormat {
+			errCorruptFormatCount++
+		} else {
+			errCount++
+		}
+	}
+
+	// Calculate read quorum.
+	readQuorum := len(formatConfigs) / 2
+
+	// Validate the err count under read quorum.
+	if errCount > len(formatConfigs)-readQuorum {
+		return errXLReadQuorum
+	}
+
+	// Check if number of corrupted format under read quorum
+	if errCorruptFormatCount > len(formatConfigs)-readQuorum {
+		return errCorruptedFormat
+	}
+
+	// Validates if format and JBOD are consistent across all disks.
+	if err = checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	// Success..
+	return nil
+}
+
+// isSavedUUIDInOrder - validates if disk uuid is present and valid in all
+// available format config JBOD. This function also validates if the disk UUID
+// is always available on all JBOD under the same order.
+func isSavedUUIDInOrder(uuid string, formatConfigs []*formatConfigV1) bool {
+	var orderIndexes []int
+	// Validate each for format.json for relevant uuid.
+	for _, formatConfig := range formatConfigs {
+		if formatConfig == nil {
+			continue
+		}
+		// Validate if UUID is present in JBOD.
+		uuidIndex := findDiskIndex(uuid, formatConfig.XL.JBOD)
+		if uuidIndex == -1 {
+			// UUID not found.
+			errorIf(errDiskNotFound, "Disk %s not found in JBOD list", uuid)
+			return false
+		}
+		// Save the position of UUID present in JBOD.
+		orderIndexes = append(orderIndexes, uuidIndex+1)
+	}
+	// Once uuid is found, verify if the uuid
+	// present in same order across all format configs.
+	prevOrderIndex := orderIndexes[0]
+	for _, orderIndex := range orderIndexes {
+		if prevOrderIndex != orderIndex {
+			errorIf(errDiskOrderMismatch, "Disk %s is in wrong order wanted %d, saw %d ", uuid, prevOrderIndex, orderIndex)
+			return false
+		}
+	}
+	// Returns success, when we have verified if uuid
+	// is consistent and valid across all format configs.
+	return true
+}
+
+// checkDisksConsistency - checks if all disks are consistent with all JBOD entries on all disks.
+func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
+	var disks = make([]string, len(formatConfigs))
+	// Collect currently available disk uuids.
+	for index, formatConfig := range formatConfigs {
+		if formatConfig == nil {
+			disks[index] = ""
+			continue
+		}
+		disks[index] = formatConfig.XL.Disk
+	}
+	// Validate collected uuids and verify JBOD.
+	for _, uuid := range disks {
+		if uuid == "" {
+			continue
+		}
+		// Is uuid present on all JBOD ?.
+		if !isSavedUUIDInOrder(uuid, formatConfigs) {
+			return fmt.Errorf("%s disk not found in JBOD", uuid)
+		}
+	}
+	return nil
+}
+
+// checkJBODConsistency - validate xl jbod order if they are consistent.
+func checkJBODConsistency(formatConfigs []*formatConfigV1) error {
+	var sentinelJBOD []string
+	// Extract first valid JBOD.
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		sentinelJBOD = format.XL.JBOD
+		break
+	}
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		currentJBOD := format.XL.JBOD
+		if !reflect.DeepEqual(sentinelJBOD, currentJBOD) {
+			return errors.New("Inconsistent JBOD found")
+		}
+	}
+	return nil
+}
+
+// findDiskIndex returns position of disk in JBOD.
+func findDiskIndex(disk string, jbod []string) int {
+	for index, uuid := range jbod {
+		if uuid == disk {
+			return index
+		}
+	}
+	return -1
+}
+
+// reorderDisks - reorder disks in JBOD order.
+func reorderDisks(bootstrapDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
+	var savedJBOD []string
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		savedJBOD = format.XL.JBOD
+		break
+	}
+	// Pick the first JBOD list to verify the order and construct new set of disk slice.
+	var newDisks = make([]StorageAPI, len(bootstrapDisks))
+	for fIndex, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		jIndex := findDiskIndex(format.XL.Disk, savedJBOD)
+		if jIndex == -1 {
+			return nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found")
+		}
+		newDisks[jIndex] = bootstrapDisks[fIndex]
+	}
+	return newDisks, nil
+}
+
+// loadFormat - loads format.json from disk.
+func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
+	buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
+	if err != nil {
+		// 'file not found' and 'volume not found' as
+		// same. 'volume not found' usually means its a fresh disk.
+		if err == errFileNotFound || err == errVolumeNotFound {
+			var vols []VolInfo
+			vols, err = disk.ListVols()
+			if err != nil {
+				return nil, err
+			}
+			if len(vols) > 1 {
+				// 'format.json' not found, but we found user data.
+				return nil, errCorruptedFormat
+			}
+			// No other data found, its a fresh disk.
+			return nil, errUnformattedDisk
+		}
+		return nil, err
+	}
+
+	// Try to decode format json into formatConfigV1 struct.
+	format = &formatConfigV1{}
+	if err = json.Unmarshal(buf, format); err != nil {
+		return nil, err
+	}
+
+	// Success.
+	return format, nil
+}
+
+// isFormatNotFound - returns true if all `format.json` are not found on all disks.
+func isFormatNotFound(formats []*formatConfigV1) bool {
+	for _, format := range formats {
+		// One of the `format.json` is found.
+		if format != nil {
+			return false
+		}
+	}
+	// All format.json missing, success.
+	return true
+}
+
+// isFormatFound - returns true if all input formats are found on all disks.
+func isFormatFound(formats []*formatConfigV1) bool {
+	for _, format := range formats {
+		// One of `format.json` is not found.
+		if format == nil {
+			return false
+		}
+	}
+	// All format.json present, success.
+	return true
+}
+
+// Heals any missing format.json on the drives. Returns error only for unexpected errors
+// as regular errors can be ignored since there might be enough quorum to be operational.
+// Heals only fresh disks.
+func healFormatXLFreshDisks(storageDisks []StorageAPI) error {
+	formatConfigs := make([]*formatConfigV1, len(storageDisks))
+	var referenceConfig *formatConfigV1
+	// Loads `format.json` from all disks.
+	for index, disk := range storageDisks {
+		// Disk not found or ignored is a valid case.
+		if disk == nil {
+			// Return nil, one of the disk is offline.
+			return nil
+		}
+		formatXL, err := loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk {
+				// format.json is missing, should be healed.
+				continue
+			} else if err == errDiskNotFound { // Is a valid case we
+				// can proceed without healing.
+				return nil
+			}
+			// Return error for unsupported errors.
+			return err
+		} // Success.
+		formatConfigs[index] = formatXL
+	}
+
+	// All `format.json` has been read successfully, previously completed.
+	if isFormatFound(formatConfigs) {
+		// Return success.
+		return nil
+	}
+
+	// All disks are fresh, format.json will be written by initFormatXL()
+	if isFormatNotFound(formatConfigs) {
+		return initFormatXL(storageDisks)
+	}
+
+	// Validate format configs for consistency in JBOD and disks.
+	if err := checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	if referenceConfig == nil {
+		// This config will be used to update the drives missing format.json.
+		for _, formatConfig := range formatConfigs {
+			if formatConfig == nil {
+				continue
+			}
+			referenceConfig = formatConfig
+			break
+		}
+	}
+
+	// Collect new JBOD.
+	newJBOD := referenceConfig.XL.JBOD
+
+	// Reorder the disks based on the JBOD order.
+	orderedDisks, err := reorderDisks(storageDisks, formatConfigs)
+	if err != nil {
+		return err
+	}
+
+	// From ordered disks fill the UUID position.
+	for index, disk := range orderedDisks {
+		if disk == nil {
+			newJBOD[index] = mustGetUUID()
+		}
+	}
+
+	// Collect new format configs.
+	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
+
+	// Collect new format configs that need to be written.
+	for index := range orderedDisks {
+		// New configs are generated since we are going
+		// to re-populate across all disks.
+		config := &formatConfigV1{
+			Version: referenceConfig.Version,
+			Format:  referenceConfig.Format,
+			XL: &xlFormat{
+				Version: referenceConfig.XL.Version,
+				Disk:    newJBOD[index],
+				JBOD:    newJBOD,
+			},
+		}
+		newFormatConfigs[index] = config
+	}
+
+	// Fill in the missing disk back from format configs.
+	// We need to make sure we have kept the previous order
+	// and allowed fresh disks to be arranged anywhere.
+	// Following block facilitates to put fresh disks.
+	for index, format := range formatConfigs {
+		// Format is missing so we go through ordered disks.
+		if format == nil {
+			// At this point when disk is missing the fresh disk
+			// in the stack get it back from storageDisks.
+			for oIndex, disk := range orderedDisks {
+				if disk == nil {
+					orderedDisks[oIndex] = storageDisks[index]
+					break
+				}
+			}
+		}
+	}
+
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolume(orderedDisks); err != nil {
+		return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
+	}
+
+	// Save new `format.json` across all disks, in JBOD order.
+	return saveFormatXL(orderedDisks, newFormatConfigs)
+}
+
+// Disks from storageDiks are put in assignedDisks if found in orderedDisks and in unAssignedDisks otherwise
+func splitDisksByUse(storageDisks, orderedDisks []StorageAPI) (assignedDisks []StorageAPI, unAssignedDisks []StorageAPI) {
+	// Populate unAssignDisks
+	for i := range storageDisks {
+		found := false
+		for j := range orderedDisks {
+			if storageDisks[i] == orderedDisks[j] {
+				found = true
+				assignedDisks = append(assignedDisks, storageDisks[i])
+				break
+			}
+		}
+		if !found {
+			unAssignedDisks = append(unAssignedDisks, storageDisks[i])
+		}
+	}
+	return assignedDisks, unAssignedDisks
+}
+
+// Inspect the content of all disks to guess the right order according to the format files.
+// The right order is represented in orderedDisks
+func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
+	for index, format := range formatConfigs {
+		if format != nil {
+			continue
+		}
+		vols, err := storageDisks[index].ListVols()
+		if err != nil {
+			return nil, err
+		}
+		if len(vols) == 0 {
+			continue
+		}
+		volName := ""
+		// Avoid picking minioMetaBucket because ListVols() returns a non ordered list
+		for i := range vols {
+			if vols[i].Name != minioMetaBucket {
+				volName = vols[i].Name
+				break
+			}
+		}
+		if volName == "" {
+			continue
+		}
+		objects, err := storageDisks[index].ListDir(volName, "")
+		if err != nil {
+			return nil, err
+		}
+		if len(objects) == 0 {
+			continue
+		}
+		xlData, err := readXLMeta(storageDisks[index], volName, objects[0])
+		if err != nil {
+			if err == errFileNotFound {
+				continue
+			}
+			return nil, err
+		}
+		diskIndex := -1
+		for i, d := range xlData.Erasure.Distribution {
+			if d == xlData.Erasure.Index {
+				diskIndex = i
+			}
+		}
+		// Check for found results
+		if diskIndex == -1 || orderedDisks[diskIndex] != nil {
+			// Some inconsistent data are found, exit immediately.
+			return nil, errCorruptedFormat
+		}
+		orderedDisks[diskIndex] = storageDisks[index]
+	}
+	return orderedDisks, nil
+}
+
+// Heals corrupted format json in all disks
+func healFormatXLCorruptedDisks(storageDisks []StorageAPI) error {
+	formatConfigs := make([]*formatConfigV1, len(storageDisks))
+	var referenceConfig *formatConfigV1
+
+	// Loads `format.json` from all disks.
+	for index, disk := range storageDisks {
+		// Disk not found or ignored is a valid case.
+		if disk == nil {
+			// Return nil, one of the disk is offline.
+			return nil
+		}
+		formatXL, err := loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk || err == errCorruptedFormat {
+				// format.json is missing or corrupted, should be healed.
+				continue
+			} else if err == errDiskNotFound { // Is a valid case we
+				// can proceed without healing.
+				return nil
+			}
+			// Return error for unsupported errors.
+			return err
+		} // Success.
+		formatConfigs[index] = formatXL
+	}
+
+	// All `format.json` has been read successfully, previously completed.
+	if isFormatFound(formatConfigs) {
+		// Return success.
+		return nil
+	}
+
+	// All disks are fresh, format.json will be written by initFormatXL()
+	if isFormatNotFound(formatConfigs) {
+		return initFormatXL(storageDisks)
+	}
+
+	// Validate format configs for consistency in JBOD and disks.
+	if err := checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	if referenceConfig == nil {
+		// This config will be used to update the drives missing format.json.
+		for _, formatConfig := range formatConfigs {
+			if formatConfig == nil {
+				continue
+			}
+			referenceConfig = formatConfig
+			break
+		}
+	}
+
+	// Collect new JBOD.
+	newJBOD := referenceConfig.XL.JBOD
+
+	// Reorder the disks based on the JBOD order.
+	orderedDisks, err := reorderDisks(storageDisks, formatConfigs)
+	if err != nil {
+		return err
+	}
+
+	// From ordered disks fill the UUID position.
+	for index, disk := range orderedDisks {
+		if disk == nil {
+			newJBOD[index] = mustGetUUID()
+		}
+	}
+
+	// For disks with corrupted formats, inspect the disks contents to guess the disks order
+	orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks, formatConfigs)
+	if err != nil {
+		return err
+	}
+
+	// At this stage, all disks with corrupted formats but with objects inside found their way.
+	// Now take care of unformatted disks, which are the `unAssignedDisks`
+	_, unAssignedDisks := splitDisksByUse(storageDisks, orderedDisks)
+
+	// Assign unassigned disks to nil elements in orderedDisks
+	for i, disk := range orderedDisks {
+		if disk == nil && len(unAssignedDisks) > 0 {
+			orderedDisks[i] = unAssignedDisks[0]
+			unAssignedDisks = unAssignedDisks[1:]
+		}
+	}
+
+	// Collect new format configs.
+	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
+
+	// Collect new format configs that need to be written.
+	for index := range orderedDisks {
+		// New configs are generated since we are going
+		// to re-populate across all disks.
+		config := &formatConfigV1{
+			Version: referenceConfig.Version,
+			Format:  referenceConfig.Format,
+			XL: &xlFormat{
+				Version: referenceConfig.XL.Version,
+				Disk:    newJBOD[index],
+				JBOD:    newJBOD,
+			},
+		}
+		newFormatConfigs[index] = config
+	}
+
+	// Save new `format.json` across all disks, in JBOD order.
+	return saveFormatXL(orderedDisks, newFormatConfigs)
+}
+
+// loadFormatXL - loads XL `format.json` and returns back properly
+// ordered storage slice based on `format.json`.
+func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageAPI, err error) {
+	var unformattedDisksFoundCnt = 0
+	var diskNotFoundCount = 0
+	var corruptedDisksFoundCnt = 0
+	formatConfigs := make([]*formatConfigV1, len(bootstrapDisks))
+
+	// Try to load `format.json` bootstrap disks.
+	for index, disk := range bootstrapDisks {
+		if disk == nil {
+			diskNotFoundCount++
+			continue
+		}
+		var formatXL *formatConfigV1
+		formatXL, err = loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk {
+				unformattedDisksFoundCnt++
+				continue
+			} else if err == errDiskNotFound {
+				diskNotFoundCount++
+				continue
+			} else if err == errCorruptedFormat {
+				corruptedDisksFoundCnt++
+				continue
+			}
+			return nil, err
+		}
+		// Save valid formats.
+		formatConfigs[index] = formatXL
+	}
+
+	// If all disks indicate that 'format.json' is not available return 'errUnformattedDisk'.
+	if unformattedDisksFoundCnt > len(bootstrapDisks)-readQuorum {
+		return nil, errUnformattedDisk
+	} else if corruptedDisksFoundCnt > len(bootstrapDisks)-readQuorum {
+		return nil, errCorruptedFormat
+	} else if diskNotFoundCount == len(bootstrapDisks) {
+		return nil, errDiskNotFound
+	} else if diskNotFoundCount > len(bootstrapDisks)-readQuorum {
+		return nil, errXLReadQuorum
+	}
+
+	// Validate the format configs read are correct.
+	if err = checkFormatXL(formatConfigs); err != nil {
+		return nil, err
+	}
+	// Erasure code requires disks to be presented in the same order each time.
+	return reorderDisks(bootstrapDisks, formatConfigs)
+}
+
+func checkFormatXLValues(formatConfigs []*formatConfigV1) error {
+	for _, formatXL := range formatConfigs {
+		if formatXL == nil {
+			continue
+		}
+		// Validate format version and format type.
+		if formatXL.Version != "1" {
+			return fmt.Errorf("Unsupported version of backend format [%s] found", formatXL.Version)
+		}
+		if formatXL.Format != "xl" {
+			return fmt.Errorf("Unsupported backend format [%s] found", formatXL.Format)
+		}
+		if formatXL.XL.Version != "1" {
+			return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version)
+		}
+		if len(formatConfigs) != len(formatXL.XL.JBOD) {
+			return fmt.Errorf("Number of disks %d did not match the backend format %d", len(formatConfigs), len(formatXL.XL.JBOD))
+		}
+	}
+	return nil
+}
+
+// checkFormatXL - verifies if format.json format is intact.
+func checkFormatXL(formatConfigs []*formatConfigV1) error {
+	if err := checkFormatXLValues(formatConfigs); err != nil {
+		return err
+	}
+	if err := checkJBODConsistency(formatConfigs); err != nil {
+		return err
+	}
+	return checkDisksConsistency(formatConfigs)
+}
+
+// saveFormatXL - populates `format.json` on disks in its order.
+func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error {
+	var errs = make([]error, len(storageDisks))
+	var wg = &sync.WaitGroup{}
+	// Write `format.json` to all disks.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int, disk StorageAPI, format *formatConfigV1) {
+			defer wg.Done()
+
+			// Marshal and write to disk.
+			formatBytes, err := json.Marshal(format)
+			if err != nil {
+				errs[index] = err
+				return
+			}
+
+			// Purge any existing temporary file, okay to ignore errors here.
+			disk.DeleteFile(minioMetaBucket, formatConfigFileTmp)
+
+			// Append file `format.json.tmp`.
+			if err = disk.AppendFile(minioMetaBucket, formatConfigFileTmp, formatBytes); err != nil {
+				errs[index] = err
+				return
+			}
+			// Rename file `format.json.tmp` --> `format.json`.
+			if err = disk.RenameFile(minioMetaBucket, formatConfigFileTmp, minioMetaBucket, formatConfigFile); err != nil {
+				errs[index] = err
+				return
+			}
+		}(index, disk, formats[index])
+	}
+
+	// Wait for the routines to finish.
+	wg.Wait()
+
+	// Validate if we encountered any errors, return quickly.
+	for _, err := range errs {
+		if err != nil {
+			// Failure.
+			return err
+		}
+	}
+
+	// Success.
+	return nil
+}
+
+// initFormatXL - save XL format configuration on all disks.
+func initFormatXL(storageDisks []StorageAPI) (err error) {
+	// Initialize jbods.
+	var jbod = make([]string, len(storageDisks))
+
+	// Initialize formats.
+	var formats = make([]*formatConfigV1, len(storageDisks))
+
+	// Initialize `format.json`.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		// Allocate format config.
+		formats[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    mustGetUUID(),
+			},
+		}
+		jbod[index] = formats[index].XL.Disk
+	}
+
+	// Update the jbod entries.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		// Save jbod.
+		formats[index].XL.JBOD = jbod
+	}
+
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolume(storageDisks); err != nil {
+		return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
+	}
+
+	// Save formats `format.json` across all disks.
+	return saveFormatXL(storageDisks, formats)
+}

cmd/fs-createfile.go

@@ -0,0 +1,39 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "io"
+
+func fsCreateFile(disk StorageAPI, reader io.Reader, buf []byte, tmpBucket, tempObj string) (int64, error) {
+	bytesWritten := int64(0)
+	// Read the buffer till io.EOF and append the read data to the temporary file.
+	for {
+		n, rErr := reader.Read(buf)
+		if rErr != nil && rErr != io.EOF {
+			return 0, traceError(rErr)
+		}
+		bytesWritten += int64(n)
+		wErr := disk.AppendFile(tmpBucket, tempObj, buf[0:n])
+		if wErr != nil {
+			return 0, traceError(wErr)
+		}
+		if rErr == io.EOF {
+			break
+		}
+	}
+	return bytesWritten, nil
+}

cmd/fs-v1-background-append.go

@@ -0,0 +1,214 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"reflect"
+	"sync"
+	"time"
+)
+
+// Error sent by appendParts go-routine when there are holes in parts.
+// For ex. let's say client uploads part-2 before part-1 in which case we
+// can not append and have to wait till part-1 is uploaded. Hence we return
+// this error. Currently this error is not used in the caller.
+var errPartsMissing = errors.New("required parts missing")
+
+// Error sent when appendParts go-routine has waited long enough and timedout.
+var errAppendPartsTimeout = errors.New("appendParts goroutine timeout")
+
+// Timeout value for the appendParts go-routine.
+var appendPartsTimeout = 24 * 60 * 60 * time.Second // 24 hours.
+
+// Holds a map of uploadID->appendParts go-routine
+type backgroundAppend struct {
+	infoMap map[string]bgAppendPartsInfo
+	sync.Mutex
+}
+
+// Input to the appendParts go-routine
+type bgAppendPartsInput struct {
+	meta  fsMetaV1   // list of parts that need to be appended
+	errCh chan error // error sent by appendParts go-routine
+}
+
+// Identifies an appendParts go-routine.
+type bgAppendPartsInfo struct {
+	inputCh    chan bgAppendPartsInput
+	timeoutCh  chan struct{} // closed by appendParts go-routine when it timesout
+	abortCh    chan struct{} // closed after abort of upload to end the appendParts go-routine
+	completeCh chan struct{} // closed after complete of upload to end the appendParts go-routine
+}
+
+// Called after a part is uploaded so that it can be appended in the background.
+func (b *backgroundAppend) append(disk StorageAPI, bucket, object, uploadID string, meta fsMetaV1) chan error {
+	b.Lock()
+	info, ok := b.infoMap[uploadID]
+	if !ok {
+		// Corresponding appendParts go-routine was not found, create a new one. Would happen when the first
+		// part of a multipart upload is uploaded.
+		inputCh := make(chan bgAppendPartsInput)
+		timeoutCh := make(chan struct{})
+		abortCh := make(chan struct{})
+		completeCh := make(chan struct{})
+
+		info = bgAppendPartsInfo{inputCh, timeoutCh, abortCh, completeCh}
+		b.infoMap[uploadID] = info
+
+		go b.appendParts(disk, bucket, object, uploadID, info)
+	}
+	b.Unlock()
+	errCh := make(chan error)
+	go func() {
+		// send input in a goroutine as send on the inputCh can block if appendParts go-routine
+		// is busy appending a part.
+		select {
+		case <-info.timeoutCh:
+			// This is to handle a rare race condition where we found info in b.infoMap
+			// but soon after that appendParts go-routine timed out.
+			errCh <- errAppendPartsTimeout
+		case info.inputCh <- bgAppendPartsInput{meta, errCh}:
+		}
+	}()
+	return errCh
+}
+
+// Called on complete-multipart-upload. Returns nil if the required parts have been appended.
+func (b *backgroundAppend) complete(disk StorageAPI, bucket, object, uploadID string, meta fsMetaV1) error {
+	b.Lock()
+	defer b.Unlock()
+	info, ok := b.infoMap[uploadID]
+	delete(b.infoMap, uploadID)
+	if !ok {
+		return errPartsMissing
+	}
+	errCh := make(chan error)
+	select {
+	case <-info.timeoutCh:
+		// This is to handle a rare race condition where we found info in b.infoMap
+		// but soon after that appendParts go-routine timedouted out.
+		return errAppendPartsTimeout
+	case info.inputCh <- bgAppendPartsInput{meta, errCh}:
+	}
+	err := <-errCh
+
+	close(info.completeCh)
+
+	return err
+}
+
+// Called after complete-multipart-upload or abort-multipart-upload so that the appendParts go-routine is not left dangling.
+func (b *backgroundAppend) abort(uploadID string) {
+	b.Lock()
+	defer b.Unlock()
+	info, ok := b.infoMap[uploadID]
+	if !ok {
+		return
+	}
+	delete(b.infoMap, uploadID)
+	info.abortCh <- struct{}{}
+}
+
+// This is run as a go-routine that appends the parts in the background.
+func (b *backgroundAppend) appendParts(disk StorageAPI, bucket, object, uploadID string, info bgAppendPartsInfo) {
+	// Holds the list of parts that is already appended to the "append" file.
+	appendMeta := fsMetaV1{}
+	// Allocate staging read buffer.
+	buf := make([]byte, readSizeV1)
+	for {
+		select {
+		case input := <-info.inputCh:
+			// We receive on this channel when new part gets uploaded or when complete-multipart sends
+			// a value on this channel to confirm if all the required parts are appended.
+			meta := input.meta
+			for {
+				// Append should be done such a way that if part-3 and part-2 is uploaded before part-1, we
+				// wait till part-1 is uploaded after which we append part-2 and part-3 as well in this for-loop.
+				part, appendNeeded := partToAppend(meta, appendMeta)
+				if !appendNeeded {
+					if reflect.DeepEqual(meta.Parts, appendMeta.Parts) {
+						// Sending nil is useful so that the complete-multipart-upload knows that
+						// all the required parts have been appended.
+						input.errCh <- nil
+					} else {
+						// Sending error is useful so that complete-multipart-upload can fall-back to
+						// its own append process.
+						input.errCh <- errPartsMissing
+					}
+					break
+				}
+				if err := appendPart(disk, bucket, object, uploadID, part, buf); err != nil {
+					disk.DeleteFile(minioMetaTmpBucket, uploadID)
+					appendMeta.Parts = nil
+					input.errCh <- err
+					break
+				}
+				appendMeta.AddObjectPart(part.Number, part.Name, part.ETag, part.Size)
+			}
+		case <-info.abortCh:
+			// abort-multipart-upload closed abortCh to end the appendParts go-routine.
+			disk.DeleteFile(minioMetaTmpBucket, uploadID)
+			close(info.timeoutCh) // So that any racing PutObjectPart does not leave a dangling go-routine.
+			return
+		case <-info.completeCh:
+			// complete-multipart-upload closed completeCh to end the appendParts go-routine.
+			close(info.timeoutCh) // So that any racing PutObjectPart does not leave a dangling go-routine.
+			return
+		case <-time.After(appendPartsTimeout):
+			// Timeout the goroutine to garbage collect its resources. This would happen if the client initiates
+			// a multipart upload and does not complete/abort it.
+			b.Lock()
+			delete(b.infoMap, uploadID)
+			b.Unlock()
+			// Delete the temporary append file as well.
+			disk.DeleteFile(minioMetaTmpBucket, uploadID)
+
+			close(info.timeoutCh)
+			return
+		}
+	}
+}
+
+// Appends the "part" to the append-file inside "tmp/" that finally gets moved to the actual location
+// upon complete-multipart-upload.
+func appendPart(disk StorageAPI, bucket, object, uploadID string, part objectPartInfo, buf []byte) error {
+	partPath := pathJoin(bucket, object, uploadID, part.Name)
+
+	offset := int64(0)
+	totalLeft := part.Size
+	for totalLeft > 0 {
+		curLeft := int64(readSizeV1)
+		if totalLeft < readSizeV1 {
+			curLeft = totalLeft
+		}
+		n, err := disk.ReadFile(minioMetaMultipartBucket, partPath, offset, buf[:curLeft])
+		if err != nil {
+			// Check for EOF/ErrUnexpectedEOF not needed as it should never happen as we know
+			// the exact size of the file and hence know the size of buf[]
+			// EOF/ErrUnexpectedEOF indicates that the length of file was shorter than part.Size and
+			// hence considered as an error condition.
+			return err
+		}
+		if err = disk.AppendFile(minioMetaTmpBucket, uploadID, buf[:n]); err != nil {
+			return err
+		}
+		offset += n
+		totalLeft -= n
+	}
+	return nil
+}