Add erasure code.

XL: Implement ListObjects channel and pool management.

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,13 @@
+## Expected behaviour
+
+## Actual behaviour
+
+## Steps to reproduce the behaviour
+
+## Minio version
+- (paste output of `minio version`)
+
+## System information
+
+
+

.gitignore

@@ -10,7 +10,8 @@ site/
 /Minio.iml
 **/access.log
 build
-isa-l
 vendor/**/*.js
 vendor/**/*.json
 release
+.DS_Store
+*.syso

.mention-bot

@@ -0,0 +1,4 @@
+{
+    "numFilesToCheck": 10,
+    "requiredOrgs": ["minio"]
+}

.travis.yml

@@ -6,16 +6,6 @@ os:
 - linux
 - osx
 
-before_install:
-  - git clone https://github.com/yasm/yasm
-  - cd yasm
-  - git checkout v1.3.0
-  - "./autogen.sh"
-  - "./configure"
-  - make
-  - export PATH=$PATH:`pwd`
-  - cd ..
-
 osx_image: xcode7.2
 
 env:

CONTRIBUTING.md

@@ -3,11 +3,11 @@
 If you do not have a working Golang environment setup please follow [Golang Installation Guide](./INSTALLGO.md).
 
 ### Setup your Minio Github Repository
-Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL and pass it to ``go get`` command. Go uses git to clone a copy into your project workspace folder.
+Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL for minio from your personal github repo (you will need it for the `git clone` command below).
 ```sh
 $ mkdir -p $GOPATH/src/github.com/minio
 $ cd $GOPATH/src/github.com/minio
-$ git clone https://github.com/$USER_ID/minio
+$ git clone <paste saved URL for personal forked minio repo>
 $ cd minio
 ```
 

Caddy.md

@@ -1,23 +1,27 @@
-### Proxy using Caddy.
+## Setting up Proxy using Caddy.
 
 Please download [Caddy Server](https://caddyserver.com/download)
 
 Create a caddy configuration file as below, change the ip addresses according to your local
 minio and DNS configuration.
 
+```bash
+$ ./minio --address localhost:9000 server <your_export_dir>
+```
+
 ```bash
 your.public.com {
-    proxy / 10.0.1.3:9000 {
+    proxy / localhost:9000 {
         proxy_header Host {host}
         proxy_header X-Real-IP {remote}
         proxy_header X-Forwarded-Proto {scheme}
     }
-    tls off
 }
 ```
 
 ```bash
 $ ./caddy
 Activating privacy features... done.
-your.public.com
+your.public.com:443
+your.public.com:80
 ```

Docker.md

@@ -1,38 +1,28 @@
-## Running Minio in Docker.
-
-### Installing Docker.
+### Run Minio docker image
 
+## Test Minio Docker Container
+Minio generates new access and secret keys each time you run this command. Container state is lost after you end this session. This mode is only intended for testing purpose.
 ```bash
-sudo apt-get install Docker.io
+docker run -p 9000:9000 minio/minio /export
 ```
 
-### Generating `minio configs` for the first time.
+## Run Minio Docker Container
+Minio container requires a persistent volume to store configuration and application data. Following command maps local persistent directories from the host OS to virtual config `~/.minio` and export `/export` directories. 
 
 ```bash
-docker run -p 9000 minio/minio:latest
+docker run -p 9000:9000 --name minio1 \
+  -v /mnt/export/minio1:/export \
+  -v /mnt/config/minio1:/root/.minio \
+  minio/minio /export
 ```
 
-### Persist `minio configs`.
-
+## Custom Access and Secret Keys
+To override Minio's auto-generated keys, you may pass secret and access keys explicitly as environment variables. Minio server also allows regular strings as access and secret keys.
 ```bash
-docker commit <running_minio_container_id> minio/my-minio
-docker stop <running_minio_container_id>
-```
-
-### Create a data volume container.
-
-```bash
-docker create -v /export --name minio-export minio/my-minio /bin/true
-```
-
-You can also map ``.minio`` directory containing authentication information.
-
-```bash
-docker create -v /export --name minio-export -v /.minio --name minio-config minio/my-minio /bin/true
-```
-
-You can then use the `--volumes-from` flag to mount the `/export` and ``/.minio`` volume in another container.
-
-```bash
-docker run -p 9000 --volumes-from minio-export --volumes-from minio-config --name minio-server1 minio/my-minio
+docker run -p 9000:9000 --name minio1 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -v /mnt/export/minio1:/export \
+  -v /mnt/config/minio1:/root/.minio \
+  minio/minio /export
 ```

Dockerfile

@@ -1,8 +1,15 @@
-# use "make dockerimage" to build
-FROM alpine:3.3
-RUN apk add --no-cache ca-certificates
-ADD minio.dockerimage /minio
-ADD export /export
+FROM golang:1.6
+
+RUN mkdir -p /go/src/app
+WORKDIR /go/src/app
+
+COPY . /go/src/app
+RUN go-wrapper download
+RUN go-wrapper install
+
+ENV ALLOW_CONTAINER_ROOT=1
+RUN mkdir -p /export/docker && cp /go/src/app/Docker.md /export/docker/
+
 EXPOSE 9000
-ENTRYPOINT ["/minio"]
-CMD ["server", "/export"]
+ENTRYPOINT ["go-wrapper", "run", "server"]
+CMD ["/export"]

FreeBSD.md

@@ -0,0 +1,156 @@
+### Minio with ZFS backend - FreeBSD
+
+This example assumes that you have a FreeBSD 10.x running
+
+#### Step *1* 
+
+As root on the FreeBSD edit `/etc/rc.conf`
+```
+zfs_enable="YES"
+```
+
+Start ZFS service
+```
+# service zfs start
+```
+
+```
+# dd if=/dev/zero of=/zfs bs=1M count=4000
+```
+
+Configure a loopback device on the `/zfs` file. 
+```
+# mdconfig -a -t vnode -f /zfs
+```
+
+Create zfs pool
+```
+# zpool create minio-example /dev/md0
+```
+
+```
+# df /minio-example
+Filesystem    512-blocks Used   Avail Capacity  Mounted on
+minio-example    7872440   38 7872402     0%    /minio-example
+```
+
+Verify if it is writable
+```
+# touch /minio-example/testfile
+# ls -l /minio-example/testfile
+-rw-r--r--  1 root  wheel  0 Apr 26 00:51 /minio-example/testfile
+```
+
+Now you have successfully created a ZFS pool for futher reading please refer to [ZFS Quickstart Guide](https://www.freebsd.org/doc/handbook/zfs-quickstart.html)
+
+However, this pool is not taking advantage of any ZFS features, so let's create a ZFS filesytem on this pool with compression enabled.  ZFS supports many compression algorithms: lzjb, gzip, zle, lz4.  LZ4 is often the most performant algorithm in terms of compression of data versus system overhead.
+
+
+```
+# zfs create minio-example/compressed-objects
+# zfs set compression=lz4 minio-example/compressed-objects
+```
+
+To keep monitoring your pool use
+
+```
+# zpool status
+  pool: minio-example
+ state: ONLINE
+  scan: none requested
+config:
+
+	NAME        STATE     READ WRITE CKSUM
+	minio-example  ONLINE       0     0     0
+	  md0       ONLINE       0     0     0
+
+errors: No known data errors
+```
+
+#### Step *2*
+
+Now start minio server on the ``/minio-example/compressed-objects``, change the permissions such that this directory is accessibly by a normal user
+
+```
+# chown -R minio-user:minio-user /minio-example/compressed-objects
+```
+
+Now login as ``minio-user`` and start minio server. 
+
+```
+$ curl https://dl.minio.io/server/minio/release/freebsd-amd64/minio > minio
+$ chmod 755 minio
+$ ./minio server /minio-example/compressed-objects
+
+AccessKey: X3RU3Q6B4T20TYB281W5  SecretKey: bvp0DF8c+6MzL60UxFqOZGlA5Z8UPfIAATUQuzLS  Region: us-east-1
+
+Minio Object Storage:
+    http://159.203.234.49:9000
+    http://10.12.0.5:9000
+    http://127.0.0.1:9000
+
+Minio Browser:
+    http://159.203.234.49:9000
+    http://10.12.0.5:9000
+    http://127.0.0.1:9000
+
+To configure Minio Client:
+    $ wget https://dl.minio.io/client/mc/release/freebsd-amd64/mc
+    $ chmod 755 mc
+    $ ./mc config host add myminio http://localhost:9000 X3RU3Q6B4T20TYB281W5 bvp0DF8c+6MzL60UxFqOZGlA5Z8UPfIAATUQuzLS
+```
+
+Point your browser to http://localhost:9000 and login with the credentials displayed on the command line. 
+
+Now you have a S3 compatible server running on top of your ZFS backend which transparently provides disk level compression for your uploaded objects.
+
+Thanks for using Minio, awaiting feedback :-) 
+
+#### Building Minio Server From Source
+
+It is possible to build the minio server from source on FreeBSD.  To do this we will used the golang distribution provided by the FreeBSD pkg infrastructure.
+
+First we will need to install golang as well as GNU make:
+
+```
+$ sudo pkg install go gmake
+Updating FreeBSD repository catalogue..
+FreeBSD repository is up-to-date.
+All repositories are up-to-date.
+The following 1 package(s) will be affected (of 0 checked):
+
+New packages to be INSTALLED:
+        go: 1.6.2,1
+        gmake: 4.1_2
+
+        The process will require 261 MiB more space.
+        40 MiB to be downloaded.
+
+        Proceed with this action? [y/N]: y
+        Fetching go-1.6.2,1.txz: 100%   40 MiB  10.4MB/s    00:04
+        Fetching gmake-4.1_2.txz: 100%  363 KiB 372.2kB/s    00:01
+        Checking integrity... done (0 conflicting)
+        [1/2] Installing go-1.6.2,1...
+        [1/2] Extracting go-1.6.2,1: 100%
+        [2/2] Installing gmake-4.1_2...
+        [2/2] Extracting gmake-4.1_2: 100%
+```
+
+Next we need to configure our environment for golang.  Insert the following lines into your ~/.profile file, and be sure to source the file before proceeding to the next step:
+
+```
+GOPATH=$HOME/golang; export GOPATH
+GOROOT=/usr/local/go/; export GOROOT
+```
+
+Now we can proceed with the normal build process of minio server as found [here](https://github.com/nomadlogic/minio/blob/master/CONTRIBUTING.md).  The only caveat is we need to specify gmake (GNU make) when building minio server as the current Makefile is not BSD make compatible:
+
+```
+$ mkdir -p $GOPATH/src/github.com/minio
+$ cd $GOPATH/src/github.com/minio
+$ git clone <paste saved URL for personal forked minio repo>
+$ cd minio
+$ gmake
+```
+
+From here you can start the server as you would with a precompiled minio server build.

Makefile

@@ -19,10 +19,14 @@ endif
 ifeq ($(CPU), x86_64)
   HOST := $(HOST)64
 else
+ifeq ($(CPU), amd64)
+  HOST := $(HOST)64
+else
 ifeq ($(CPU), i686)
   HOST := $(HOST)32
 endif
 endif
+endif
 
 
 #############################################
@@ -41,6 +45,10 @@ ifndef (OS)
   else
   ifeq ($(HOST), Darwin32)
     arch = clang
+  else
+  ifeq ($(HOST), FreeBSD64)
+    arch = gcc
+  endif
   endif
   endif
   endif
@@ -60,12 +68,12 @@ checkdocker:
 
 getdeps: checks
 	@go get -u github.com/golang/lint/golint && echo "Installed golint:"
-	@go get -u golang.org/x/tools/cmd/vet && echo "Installed vet:"
 	@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
 	@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:"
 	@go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell:"
+	@go get -u github.com/gordonklaus/ineffassign && echo "Installed ineffassign:"
 
-verifiers: getdeps vet fmt lint cyclo spelling
+verifiers: vet fmt lint cyclo spelling
 
 vet:
 	@echo "Running $@:"
@@ -79,25 +87,21 @@ fmt:
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
 
-## Configure Intel library.
-isa-l:
-
-	@echo "Configuring $@:"
-	@git clone -q https://github.com/minio/isa-l.git
-	@cd isa-l; make -f Makefile.unx arch=$(arch) >/dev/null; mv include isa-l;
-
 lint:
 	@echo "Running $@:"
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint github.com/minio/minio/pkg...
 
+ineffassign:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/ineffassign .
+
 cyclo:
 	@echo "Running $@:"
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
 
-build: getdeps verifiers $(UI_ASSETS) isa-l
-	@GO15VENDOREXPERIMENT=1 go generate github.com/minio/minio/pkg/crypto/sha1
+build: getdeps verifiers $(UI_ASSETS)
 
 deadcode:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
@@ -108,12 +112,12 @@ spelling:
 
 test: build
 	@echo "Running all minio testing:"
-	@GODEBUG=cgocheck=0 CGO_CPPFLAGS="-I$(PWD)/isa-l" CGO_LDFLAGS="$(PWD)/isa-l/isa-l.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
-	@GODEBUG=cgocheck=0 CGO_CPPFLAGS="-I$(PWD)/isa-l" CGO_LDFLAGS="$(PWD)/isa-l/isa-l.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
 
 gomake-all: build
 	@echo "Installing minio:"
-	@CGO_CPPFLAGS="-I$(PWD)/isa-l" CGO_LDFLAGS="$(PWD)/isa-l/isa-l.a" GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
+	@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
 
 pkg-add:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
@@ -129,19 +133,18 @@ pkg-list:
 
 install: gomake-all
 
-dockerimage: checkdocker verifiers $(UI_ASSETS)
+dockerimage: checkdocker getdeps verifiers $(UI_ASSETS)
 	@echo "Building docker image:" minio:$(TAG)
-	@GO15VENDOREXPERIMENT=1 GOOS=linux GOARCH=amd64 go build --ldflags $(DOCKER_LDFLAGS) -o minio.dockerimage
-	@mkdir -p export
-	@sudo docker build --rm --tag=minio/minio:$(TAG) .
-	@rmdir export
-	@rm minio.dockerimage
+	@GO15VENDOREXPERIMENT=1 GOOS=linux GOARCH=amd64 go build --ldflags $(DOCKER_LDFLAGS) -o docker/minio.dockerimage
+	@cd docker; mkdir -p export; sudo docker build --rm --tag=minio/minio:$(TAG) .
+	@rmdir docker/export
+	@rm docker/minio.dockerimage
 
 release: verifiers
-	@MINIO_RELEASE=RELEASE GO15VENDOREXPERIMENT=1 ./buildscripts/build.sh
+	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
 
 experimental: verifiers
-	@MINIO_RELEASE=EXPERIMENTAL GO15VENDOREXPERIMENT=1 ./buildscripts/build.sh
+	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
 
 clean:
 	@echo "Cleaning up all the generated files:"

README.md

@@ -2,16 +2,15 @@
 
 Minio is an object storage server compatible with Amazon S3 and licensed under [Apache license 2.0](./LICENSE).
 
-
 ## Description
 
 Minio is an open source object storage server released under Apache License V2. It is compatible with Amazon S3 cloud storage service. Minio follows a minimalist design philosophy.
-Minio is light enough to be bundled with the application stack. It sits on the side of NodeJS, Redis, MySQL and the likes. Unlike databases, Minio stores objects such as photos, videos, log files, backups, container / VM images and so on. Minio is best suited for storing blobs of information ranging from KBs to TBs each. In a simplistic sense, it is like a FTP server with a simple get / put API over HTTP.
+Minio is light enough to be bundled with the application stack. It sits on the side of NodeJS, Redis, MySQL and the likes. Unlike databases, Minio stores objects such as photos, videos, log files, backups, container / VM images and so on. Minio is best suited for storing blobs of information ranging from KBs to 5 TBs each. In a simplistic sense, it is like a FTP server with a simple get / put API over HTTP.
 
 Minio currently implements two backends
 
-  - Filesystem (FS) - is available and ready for general purpose use. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.
-  - ErasureCoded (XL) - is work in progress and not ready for general purpose use.
+  - Filesystem (FS) - is available and ready for general purpose use.
+  - ErasureCoded (XL) - is available, but it is not ready for general purpose use.
 
 ## Minio Client
 
@@ -59,6 +58,16 @@ Download ``minio`` for:
 C:\Users\Username\Downloads> minio.exe --help
 ~~~
 
+#### FreeBSD
+
+Download ``minio`` from https://dl.minio.io/server/minio/release/freebsd-amd64/minio
+
+~~~
+$ chmod 755 minio
+$ ./minio --help
+~~~
+Read more here on [How to configure Minio on FreeBSD with ZFS backend.](./FreeBSD.md)
+
 #### Docker container
 
 Download ``minio`` for docker.
@@ -71,7 +80,7 @@ Read more here on [How to configure data volume containers for Minio?](./Docker.
 
 #### Source
 <blockquote>
-NOTE: Source installation is intended for only developers and advanced users.  For general use, please download official releases from https://minio.io/download.
+NOTE: Source installation is intended for only developers and advanced users.  For general use, please download official releases from https://minio.io/downloads.
 </blockquote>
 
 If you do not have a working Golang environment, please follow [Install Golang](./INSTALLGO.md).
@@ -84,46 +93,20 @@ $ make
 
 ### How to use Minio?
 
-```
-NAME:
-  minio server - Start Minio cloud storage server.
-
-USAGE:
-  minio server [OPTION VALUE] PATH
-
-  OPTION = expiry        VALUE = NN[h|m|s] [DEFAULT=Unlimited]
-  OPTION = min-free-disk VALUE = NN% [DEFAULT: 10%]
-
-EXAMPLES:
-  1. Start minio server on Linux.
-        $ minio server /home/shared
-
-  2. Start minio server on Windows.
-        $ minio server C:\MyShare
-
-  3. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
-        $ minio --address 192.168.1.101:9000 server /home/shared
-
-  4. Start minio server with minimum free disk threshold to 5%
-        $ minio server min-free-disk 5% /home/shared/Pictures
-
-  5. Start minio server with minimum free disk threshold to 15% with auto expiration set to 1h
-        $ minio server min-free-disk 15% expiry 1h /home/shared/Documents
-```
-
-#### Start Minio server.
+Start minio server.
 
 ~~~
 $ minio server ~/Photos
+
 AccessKey: WLGDGYAQYIGI833EV05A  SecretKey: BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF  Region: us-east-1
 
 Minio Object Storage:
     http://127.0.0.1:9000
-    http://10.0.0.3:9000
+    http://10.1.10.177:9000
 
 Minio Browser:
     http://127.0.0.1:9000
-    http://10.0.0.3:9000
+    http://10.1.10.177:9000
 
 To configure Minio Client:
     $ wget https://dl.minio.io/client/mc/release/darwin-amd64/mc
@@ -156,16 +139,41 @@ $ aws configure set default.s3.signature_version s3v4
 To list your buckets.
 ```
 $ aws --endpoint-url http://localhost:9000 s3 ls
-2016-01-07 16:38:23 testbucket
+2016-03-27 02:06:30 deebucket
+2016-03-28 21:53:49 guestbucket
+2016-03-29 13:34:34 mbtest
+2016-03-26 22:01:36 mybucket
+2016-03-26 15:37:02 testbucket
 ```
 
 To list contents inside bucket.
 ```
-$ aws --endpoint-url http://localhost:9000 s3 ls s3://testbucket
-                           PRE test/
-2015-12-17 08:46:41   12232928 vim
-2016-01-07 16:38:23   32232928 emacs
-2015-12-09 08:05:24     138504 s3cmd
+$ aws --endpoint-url http://localhost:9000 s3 ls s3://mybucket
+2016-03-30 00:26:53      69297 argparse-1.2.1.tar.gz
+2016-03-30 00:35:37      67250 simplejson-3.3.0.tar.gz
+```
+
+To make a bucket.
+```
+$ aws --endpoint-url http://localhost:9000 s3 mb s3://mybucket
+make_bucket: s3://mybucket/
+```
+
+To add an object to a bucket.
+```
+$ aws --endpoint-url http://localhost:9000 s3 cp simplejson-3.3.0.tar.gz s3://mybucket
+upload: ./simplejson-3.3.0.tar.gz to s3://mybucket/simplejson-3.3.0.tar.gz
+```
+
+Delete an object from a bucket.
+```
+$ aws --endpoint-url http://localhost:9000 s3 rm s3://mybucket/argparse-1.2.1.tar.gzdelete: s3://mybucket/argparse-1.2.1.tar.gz
+```
+
+Remove a bucket.
+```
+$ aws --endpoint-url http://localhost:9000 s3 rb s3://mybucket
+remove_bucket: s3://mybucket/
 ```
 
 #### How to use AWS SDK with Minio?
@@ -185,21 +193,65 @@ host_base = localhost:9000
 host_bucket = localhost:9000
 access_key = YOUR_ACCESS_KEY_HERE
 secret_key = YOUR_SECRET_KEY_HERE
+signature_v2 = False
+bucket_location = us-east-1
 ```
 
-To list your buckets.
+To make a bucket.
+```
+$ s3cmd mb s3://mybucket
+Bucket 's3://mybucket/' created
+```
+
+To copy an object to bucket.
+```
+$ s3cmd put newfile s3://testbucket
+upload: 'newfile' -> 's3://testbucket/newfile'  
+```
+
+To copy an object to local system.
+```
+$ s3cmd get s3://testbucket/newfile
+download: 's3://testbucket/newfile' -> './newfile'
+```
+
+To sync local file/directory to a bucket.
+```
+$ s3cmd sync newdemo s3://testbucket
+upload: 'newdemo/newdemofile.txt' -> 's3://testbucket/newdemo/newdemofile.txt'
+```
+
+To sync bucket or object with local filesystem.
+```
+$ s3cmd sync  s3://otherbucket otherlocalbucket
+download: 's3://otherbucket/cat.jpg' -> 'otherlocalbucket/cat.jpg'
+```
+
+To list buckets.
 ```
 $ s3cmd ls s3://
 2015-12-09 16:12  s3://testbbucket
 ```
 
-To list contents inside buckets.
+To list contents inside bucket.
 ```
 $ s3cmd ls s3://testbucket/
                        DIR   s3://testbucket/test/
 2015-12-09 16:05    138504   s3://testbucket/newfile
 ```
 
+Delete an object from bucket.
+```
+$ s3cmd del s3://testbucket/newfile
+delete: 's3://testbucket/newfile'
+```
+
+Delete a bucket.
+```
+$ s3cmd rb s3://testbucket
+Bucket 's3://testbucket/' removed
+```
+
 ## Contribute to Minio Project
 Please follow Minio [Contributor's Guide](./CONTRIBUTING.md)
 

access-key.go

@@ -0,0 +1,94 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"regexp"
+)
+
+// credential container for access and secret keys.
+type credential struct {
+	AccessKeyID     string `json:"accessKey"`
+	SecretAccessKey string `json:"secretKey"`
+}
+
+// stringer colorized access keys.
+func (a credential) String() string {
+	accessStr := colorMagenta("AccessKey: ") + colorWhite(a.AccessKeyID)
+	secretStr := colorMagenta("SecretKey: ") + colorWhite(a.SecretAccessKey)
+	return fmt.Sprint(accessStr + "  " + secretStr)
+}
+
+const (
+	minioAccessID = 20
+	minioSecretID = 40
+)
+
+// isValidSecretKey - validate secret key.
+var isValidSecretKey = regexp.MustCompile(`^.{8,40}$`)
+
+// isValidAccessKey - validate access key.
+var isValidAccessKey = regexp.MustCompile(`^[a-zA-Z0-9\\-\\.\\_\\~]{5,20}$`)
+
+// mustGenAccessKeys - must generate access credentials.
+func mustGenAccessKeys() (creds credential) {
+	creds, err := genAccessKeys()
+	fatalIf(err, "Unable to generate access keys.")
+	return creds
+}
+
+// genAccessKeys - generate access credentials.
+func genAccessKeys() (credential, error) {
+	accessKeyID, err := genAccessKeyID()
+	if err != nil {
+		return credential{}, err
+	}
+	secretAccessKey, err := genSecretAccessKey()
+	if err != nil {
+		return credential{}, err
+	}
+	creds := credential{
+		AccessKeyID:     string(accessKeyID),
+		SecretAccessKey: string(secretAccessKey),
+	}
+	return creds, nil
+}
+
+// genAccessKeyID - generate random alpha numeric value using only uppercase characters
+// takes input as size in integer
+func genAccessKeyID() ([]byte, error) {
+	alpha := make([]byte, minioAccessID)
+	if _, err := rand.Read(alpha); err != nil {
+		return nil, err
+	}
+	for i := 0; i < minioAccessID; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return alpha, nil
+}
+
+// genSecretAccessKey - generate random base64 numeric value from a random seed.
+func genSecretAccessKey() ([]byte, error) {
+	rb := make([]byte, minioSecretID)
+	if _, err := rand.Read(rb); err != nil {
+		return nil, err
+	}
+	return []byte(base64.StdEncoding.EncodeToString(rb))[:minioSecretID], nil
+}

accesslog-handler.go

@@ -22,8 +22,6 @@ import (
 	"net/url"
 	"os"
 	"time"
-
-	"github.com/minio/minio/pkg/probe"
 )
 
 type accessLogHandler struct {
@@ -60,14 +58,14 @@ type LogMessage struct {
 
 func (h *accessLogHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	message, err := getLogMessage(w, req)
-	fatalIf(err.Trace(), "Unable to extract http message.", nil)
-	_, e := h.accessLogFile.Write(message)
-	fatalIf(probe.NewError(e), "Writing to log file failed.", nil)
+	fatalIf(err, "Unable to parse HTTP request and response fields.")
+	_, err = h.accessLogFile.Write(message)
+	fatalIf(err, "Unable to log HTTP access.")
 
 	h.Handler.ServeHTTP(w, req)
 }
 
-func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, *probe.Error) {
+func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, error) {
 	logMessage := &LogMessage{
 		StartTime: time.Now().UTC(),
 	}
@@ -103,9 +101,9 @@ func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, *probe.Err
 
 	// logMessage.HTTP.Request = req
 	logMessage.Duration = time.Now().UTC().Sub(logMessage.StartTime)
-	js, e := json.Marshal(logMessage)
-	if e != nil {
-		return nil, probe.NewError(e)
+	js, err := json.Marshal(logMessage)
+	if err != nil {
+		return nil, err
 	}
 	js = append(js, byte('\n')) // append a new line
 	return js, nil
@@ -113,8 +111,8 @@ func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, *probe.Err
 
 // setAccessLogHandler logs requests
 func setAccessLogHandler(h http.Handler) http.Handler {
-	file, e := os.OpenFile("access.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
-	fatalIf(probe.NewError(e), "Unable to open access log.", nil)
+	file, err := os.OpenFile("access.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	fatalIf(err, "Failed top open access log.")
 
 	return &accessLogHandler{Handler: h, accessLogFile: file}
 }

api-auth-utils.go

@@ -1,82 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"crypto/rand"
-	"encoding/base64"
-	"regexp"
-
-	"github.com/minio/minio/pkg/probe"
-)
-
-const (
-	minioAccessID = 20
-	minioSecretID = 40
-)
-
-// isValidAccessKey - validate access key
-func isValidAccessKey(accessKeyID string) bool {
-	if accessKeyID == "" {
-		return true
-	}
-	regex := regexp.MustCompile("^[A-Z0-9\\-\\.\\_\\~]{20}$")
-	return regex.MatchString(accessKeyID)
-}
-
-// isValidSecretKey - validate secret key
-func isValidSecretKey(secretKeyID string) bool {
-	regex := regexp.MustCompile("^[a-zA-Z0-9\\-\\.\\_\\~]{40}$")
-	return regex.MatchString(secretKeyID)
-}
-
-// generateAccessKeyID - generate random alpha numeric value using only uppercase characters
-// takes input as size in integer
-func generateAccessKeyID() ([]byte, *probe.Error) {
-	alpha := make([]byte, minioAccessID)
-	if _, e := rand.Read(alpha); e != nil {
-		return nil, probe.NewError(e)
-	}
-	for i := 0; i < minioAccessID; i++ {
-		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
-	}
-	return alpha, nil
-}
-
-// generateSecretAccessKey - generate random base64 numeric value from a random seed.
-func generateSecretAccessKey() ([]byte, *probe.Error) {
-	rb := make([]byte, minioSecretID)
-	if _, e := rand.Read(rb); e != nil {
-		return nil, probe.NewError(e)
-	}
-	return []byte(base64.StdEncoding.EncodeToString(rb))[:minioSecretID], nil
-}
-
-// mustGenerateAccessKeyID - must generate random alpha numeric value using only uppercase characters
-// takes input as size in integer
-func mustGenerateAccessKeyID() []byte {
-	alpha, err := generateAccessKeyID()
-	fatalIf(err.Trace(), "Unable to generate accessKeyID.", nil)
-	return alpha
-}
-
-// mustGenerateSecretAccessKey - generate random base64 numeric value from a random seed.
-func mustGenerateSecretAccessKey() []byte {
-	secretKey, err := generateSecretAccessKey()
-	fatalIf(err.Trace(), "Unable to generate secretAccessKey.", nil)
-	return secretKey
-}

api-datatypes.go

@@ -16,11 +16,22 @@
 
 package main
 
+import (
+	"encoding/xml"
+)
+
 // ObjectIdentifier carries key name for the object to delete.
 type ObjectIdentifier struct {
 	ObjectName string `xml:"Key"`
 }
 
+// createBucketConfiguration container for bucket configuration request from client.
+// Used for parsing the location from the request body for MakeBucketbucket.
+type createBucketLocationConfiguration struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"`
+	Location string   `xml:"LocationConstraint"`
+}
+
 // DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
 type DeleteObjectsRequest struct {
 	// Element to enable quiet mode for the request

api-errors.go

@@ -83,11 +83,32 @@ const (
 	ErrMalformedPOSTRequest
 	ErrSignatureVersionNotSupported
 	ErrBucketNotEmpty
-	ErrRootPathFull
-	ErrObjectExistsAsPrefix
 	ErrAllAccessDisabled
 	ErrMalformedPolicy
+	ErrMissingFields
+	ErrMissingCredTag
+	ErrCredMalformed
+	ErrInvalidRegion
+	ErrInvalidService
+	ErrInvalidRequestVersion
+	ErrMissingSignTag
+	ErrMissingSignHeadersTag
+	ErrPolicyAlreadyExpired
+	ErrMalformedDate
+	ErrMalformedExpires
+	ErrAuthHeaderEmpty
+	ErrExpiredPresignRequest
+	ErrMissingDateHeader
+	ErrInvalidQuerySignatureAlgo
+	ErrInvalidQueryParams
+	ErrBucketAlreadyOwnedByYou
 	// Add new error codes here.
+
+	// Minio extended errors.
+	ErrReadQuorum
+	ErrWriteQuorum
+	ErrStorageFull
+	ErrObjectExistsAsDirectory
 )
 
 // error code to APIError structure, these fields carry respective
@@ -278,16 +299,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The bucket you tried to delete is not empty.",
 		HTTPStatusCode: http.StatusConflict,
 	},
-	ErrRootPathFull: {
-		Code:           "RootPathFull",
-		Description:    "Root path has reached its minimum free disk threshold. Please delete few objects to proceed.",
-		HTTPStatusCode: http.StatusInternalServerError,
-	},
-	ErrObjectExistsAsPrefix: {
-		Code:           "ObjectExistsAsPrefix",
-		Description:    "An object already exists as your prefix, choose a different prefix to proceed.",
-		HTTPStatusCode: http.StatusConflict,
-	},
 	ErrAllAccessDisabled: {
 		Code:           "AllAccessDisabled",
 		Description:    "All access to this bucket has been disabled.",
@@ -295,12 +306,166 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrMalformedPolicy: {
 		Code:           "MalformedPolicy",
-		Description:    "Policy has invalid resource",
+		Description:    "Policy has invalid resource.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMissingFields: {
+		Code:           "MissingFields",
+		Description:    "Missing fields in request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingCredTag: {
+		Code:           "InvalidRequest",
+		Description:    "Missing Credential field for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrCredMalformed: {
+		Code:           "CredentialMalformed",
+		Description:    "Credential field malformed does not follow accessKeyID/credScope.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedDate: {
+		Code:           "MalformedDate",
+		Description:    "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRegion: {
+		Code:           "InvalidRegion",
+		Description:    "Region does not match.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidService: {
+		Code:           "AccessDenied",
+		Description:    "Service scope should be of value 's3'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRequestVersion: {
+		Code:           "AccessDenied",
+		Description:    "Request scope should be of value 'aws4_request'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignTag: {
+		Code:           "AccessDenied",
+		Description:    "Signature header missing Signature field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignHeadersTag: {
+		Code:           "InvalidArgument",
+		Description:    "Signature header missing SignedHeaders field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPolicyAlreadyExpired: {
+		Code:           "AccessDenied",
+		Description:    "Invalid according to Policy: Policy expired.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedExpires: {
+		Code:           "MalformedExpires",
+		Description:    "Malformed expires header, expected non-zero number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthHeaderEmpty: {
+		Code:           "InvalidArgument",
+		Description:    "Authorization header is invalid -- one and only one ' ' (space) required.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingDateHeader: {
+		Code:           "AccessDenied",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQuerySignatureAlgo: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrExpiredPresignRequest: {
+		Code:           "AccessDenied",
+		Description:    "Request has expired.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQueryParams: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketAlreadyOwnedByYou: {
+		Code:           "BucketAlreadyOwnedByYou",
+		Description:    "Your previous request to create the named bucket succeeded and you already own it.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	/// Minio extensions.
+	ErrStorageFull: {
+		Code:           "XMinioStorageFull",
+		Description:    "Storage backend has reached its minimum free disk threshold. Please delete few objects to proceed.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	ErrObjectExistsAsDirectory: {
+		Code:           "XMinioObjectExistsAsDirectory",
+		Description:    "Object name already exists as a directory.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrReadQuorum: {
+		Code:           "XMinioReadQuorum",
+		Description:    "Multiple disk failures, unable to reconstruct data.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrWriteQuorum: {
+		Code:           "XMinioWriteQuorum",
+		Description:    "Multiple disks failures, unable to write data.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
 	// Add your error structure here.
 }
 
+// toAPIErrorCode - Converts embedded errors. Convenience
+// function written to handle all cases where we have known types of
+// errors returned by underlying layers.
+func toAPIErrorCode(err error) (apiErr APIErrorCode) {
+	if err == nil {
+		return ErrNone
+	}
+	// Verify if the underlying error is signature mismatch.
+	if err == errSignatureMismatch {
+		return ErrSignatureDoesNotMatch
+	}
+	switch err.(type) {
+	case StorageFull:
+		apiErr = ErrStorageFull
+	case BadDigest:
+		apiErr = ErrBadDigest
+	case IncompleteBody:
+		apiErr = ErrIncompleteBody
+	case ObjectExistsAsDirectory:
+		apiErr = ErrObjectExistsAsDirectory
+	case BucketNameInvalid:
+		apiErr = ErrInvalidBucketName
+	case BucketNotFound:
+		apiErr = ErrNoSuchBucket
+	case BucketNotEmpty:
+		apiErr = ErrBucketNotEmpty
+	case BucketExists:
+		apiErr = ErrBucketAlreadyOwnedByYou
+	case ObjectNotFound:
+		apiErr = ErrNoSuchKey
+	case ObjectNameInvalid:
+		apiErr = ErrNoSuchKey
+	case InvalidUploadID:
+		apiErr = ErrNoSuchUpload
+	case InvalidPart:
+		apiErr = ErrInvalidPart
+	case InsufficientWriteQuorum:
+		apiErr = ErrWriteQuorum
+	case InsufficientReadQuorum:
+		apiErr = ErrReadQuorum
+	case PartTooSmall:
+		apiErr = ErrEntityTooSmall
+	default:
+		apiErr = ErrInternalError
+	}
+	return apiErr
+}
+
 // getAPIError provides API Error for input API error code.
 func getAPIError(code APIErrorCode) APIError {
 	return errorCodeResponse[code]

api-headers.go

@@ -23,8 +23,6 @@ import (
 	"net/http"
 	"runtime"
 	"strconv"
-
-	"github.com/minio/minio/pkg/fs"
 )
 
 //// helpers
@@ -60,20 +58,20 @@ func encodeResponse(response interface{}) []byte {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, metadata fs.ObjectMetadata, contentRange *httpRange) {
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
 	// set common headers
 	setCommonHeaders(w)
 
 	// set object-related metadata headers
-	lastModified := metadata.LastModified.UTC().Format(http.TimeFormat)
+	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
 	w.Header().Set("Last-Modified", lastModified)
 
-	w.Header().Set("Content-Type", metadata.ContentType)
-	if metadata.MD5 != "" {
-		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
+	w.Header().Set("Content-Type", objInfo.ContentType)
+	if objInfo.MD5Sum != "" {
+		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
 	}
 
-	w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
+	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
 
 	// for providing ranged content
 	if contentRange != nil {

api-resources.go

@@ -19,36 +19,78 @@ package main
 import (
 	"net/url"
 	"strconv"
-
-	"github.com/minio/minio/pkg/fs"
 )
 
-// parse bucket url queries
-func getBucketResources(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
+// Parse bucket url queries
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
 	prefix = values.Get("prefix")
 	marker = values.Get("marker")
 	delimiter = values.Get("delimiter")
-	maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
 	encodingType = values.Get("encoding-type")
 	return
 }
 
-// part bucket url queries for ?uploads
-func getBucketMultipartResources(values url.Values) (v fs.BucketMultipartResourcesMetadata) {
-	v.Prefix = values.Get("prefix")
-	v.KeyMarker = values.Get("key-marker")
-	v.MaxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
-	v.Delimiter = values.Get("delimiter")
-	v.EncodingType = values.Get("encoding-type")
-	v.UploadIDMarker = values.Get("upload-id-marker")
+// Parse bucket url queries for ListObjects V2.
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, maxkeys int, encodingType string) {
+	prefix = values.Get("prefix")
+	startAfter = values.Get("start-after")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
+	encodingType = values.Get("encoding-type")
+	token = values.Get("continuation-token")
 	return
 }
 
-// parse object url queries
-func getObjectResources(values url.Values) (v fs.ObjectResourcesMetadata) {
-	v.UploadID = values.Get("uploadId")
-	v.PartNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
-	v.MaxParts, _ = strconv.Atoi(values.Get("max-parts"))
-	v.EncodingType = values.Get("encoding-type")
+// Parse bucket url queries
+func getBucketResources(values url.Values) (listType int, prefix, marker, delimiter string, maxkeys int, encodingType string) {
+	if values.Get("list-type") != "" {
+		listType, _ = strconv.Atoi(values.Get("list-type"))
+	}
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse bucket url queries for ?uploads
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+	prefix = values.Get("prefix")
+	keyMarker = values.Get("key-marker")
+	uploadIDMarker = values.Get("upload-id-marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-uploads") != "" {
+		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
+	} else {
+		maxUploads = maxUploadsList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse object url queries
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
+	uploadID = values.Get("uploadId")
+	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+	if values.Get("max-parts") != "" {
+		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+	} else {
+		maxParts = maxPartsList
+	}
+	encodingType = values.Get("encoding-type")
 	return
 }

api-response.go

@@ -20,15 +20,13 @@ import (
 	"encoding/xml"
 	"net/http"
 	"time"
-
-	"github.com/minio/minio/pkg/fs"
 )
 
 const (
-	// Reply date format
-	timeFormatAMZ = "2006-01-02T15:04:05.000Z"
-	// Limit number of objects in a given response.
-	maxObjectList = 1000
+	timeFormatAMZ  = "2006-01-02T15:04:05.000Z" // Reply date format
+	maxObjectList  = 1000                       // Limit number of objects in a listObjectsResponse.
+	maxUploadsList = 1000                       // Limit number of uploads in a listUploadsResponse.
+	maxPartsList   = 1000                       // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.
@@ -37,28 +35,6 @@ type LocationResponse struct {
 	Location string   `xml:",chardata"`
 }
 
-// AccessControlPolicyResponse - format for get bucket acl response.
-type AccessControlPolicyResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
-
-	AccessControlList struct {
-		Grants []Grant `xml:"Grant"`
-	}
-	Owner Owner
-}
-
-// Grant container for grantee and permission.
-type Grant struct {
-	Grantee struct {
-		ID           string
-		DisplayName  string
-		EmailAddress string
-		Type         string
-		URI          string
-	}
-	Permission string
-}
-
 // ListObjectsResponse - format for list objects response.
 type ListObjectsResponse struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
@@ -89,6 +65,37 @@ type ListObjectsResponse struct {
 	Prefix     string
 }
 
+// ListObjectsV2Response - format for list objects response.
+type ListObjectsV2Response struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+
+	CommonPrefixes []CommonPrefix
+	Contents       []Object
+
+	Delimiter string
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string
+
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+	StartAfter  string
+	MaxKeys     int
+	Name        string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	ContinuationToken     string
+	NextContinuationToken string
+	Prefix                string
+}
+
 // Part container for part metadata.
 type Part struct {
 	PartNumber   int
@@ -218,6 +225,16 @@ type CompleteMultipartUploadResponse struct {
 	ETag     string
 }
 
+// PostResponse container for completed post upload response
+type PostResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PostResponse" json:"-"`
+
+	Location string
+	Bucket   string
+	Key      string
+	ETag     string
+}
+
 // DeleteError structure.
 type DeleteError struct {
 	Code    string
@@ -241,13 +258,18 @@ func getLocation(r *http.Request) string {
 	return r.URL.Path
 }
 
+// getObjectLocation gets the relative URL for an object
+func getObjectLocation(bucketName string, key string) string {
+	return "/" + bucketName + "/" + key
+}
+
 // takes an array of Bucketmetadata information for serialization
 // input:
 // array of bucket metadata
 //
 // output:
 // populated struct that can be serialized to match xml and json api spec output
-func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsResponse {
+func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	var listbuckets []Bucket
 	var data = ListBucketsResponse{}
 	var owner = Owner{}
@@ -269,7 +291,7 @@ func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsRespons
 }
 
 // generates an ListObjects response for the said bucket with other enumerated options.
-func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp fs.ListObjectsResult) ListObjectsResponse {
+func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	var contents []Object
 	var prefixes []CommonPrefix
 	var owner = Owner{}
@@ -280,13 +302,13 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe
 
 	for _, object := range resp.Objects {
 		var content = Object{}
-		if object.Object == "" {
+		if object.Name == "" {
 			continue
 		}
-		content.Key = object.Object
-		content.LastModified = object.LastModified.UTC().Format(timeFormatAMZ)
-		if object.MD5 != "" {
-			content.ETag = "\"" + object.MD5 + "\""
+		content.Key = object.Name
+		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZ)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
 		}
 		content.Size = object.Size
 		content.StorageClass = "STANDARD"
@@ -313,6 +335,51 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe
 	return data
 }
 
+// generates an ListObjects response for the said bucket with other enumerated options.
+func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
+	var contents []Object
+	var prefixes []CommonPrefix
+	var owner = Owner{}
+	var data = ListObjectsV2Response{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, object := range resp.Objects {
+		var content = Object{}
+		if object.Name == "" {
+			continue
+		}
+		content.Key = object.Name
+		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZ)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
+		}
+		content.Size = object.Size
+		content.StorageClass = "STANDARD"
+		content.Owner = owner
+		contents = append(contents, content)
+	}
+	// TODO - support EncodingType in xml decoding
+	data.Name = bucket
+	data.Contents = contents
+
+	data.StartAfter = startAfter
+	data.Delimiter = delimiter
+	data.Prefix = prefix
+	data.MaxKeys = maxKeys
+	data.ContinuationToken = token
+	data.NextContinuationToken = resp.NextMarker
+	data.IsTruncated = resp.IsTruncated
+	for _, prefix := range resp.Prefixes {
+		var prefixItem = CommonPrefix{}
+		prefixItem.Prefix = prefix
+		prefixes = append(prefixes, prefixItem)
+	}
+	data.CommonPrefixes = prefixes
+	return data
+}
+
 // generateCopyObjectResponse
 func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
 	return CopyObjectResponse{
@@ -341,56 +408,61 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string)
 }
 
 // generateListPartsResult
-func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPartsResponse {
+func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
 	// TODO - support EncodingType in xml decoding
 	listPartsResponse := ListPartsResponse{}
-	listPartsResponse.Bucket = objectMetadata.Bucket
-	listPartsResponse.Key = objectMetadata.Object
-	listPartsResponse.UploadID = objectMetadata.UploadID
+	listPartsResponse.Bucket = partsInfo.Bucket
+	listPartsResponse.Key = partsInfo.Object
+	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = "STANDARD"
 	listPartsResponse.Initiator.ID = "minio"
 	listPartsResponse.Initiator.DisplayName = "minio"
 	listPartsResponse.Owner.ID = "minio"
 	listPartsResponse.Owner.DisplayName = "minio"
 
-	listPartsResponse.MaxParts = objectMetadata.MaxParts
-	listPartsResponse.PartNumberMarker = objectMetadata.PartNumberMarker
-	listPartsResponse.IsTruncated = objectMetadata.IsTruncated
-	listPartsResponse.NextPartNumberMarker = objectMetadata.NextPartNumberMarker
+	listPartsResponse.MaxParts = partsInfo.MaxParts
+	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
+	listPartsResponse.IsTruncated = partsInfo.IsTruncated
+	listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
 
-	listPartsResponse.Parts = make([]Part, len(objectMetadata.Part))
-	for _, part := range objectMetadata.Part {
+	listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
+	for index, part := range partsInfo.Parts {
 		newPart := Part{}
 		newPart.PartNumber = part.PartNumber
 		newPart.ETag = "\"" + part.ETag + "\""
 		newPart.Size = part.Size
 		newPart.LastModified = part.LastModified.UTC().Format(timeFormatAMZ)
-		listPartsResponse.Parts = append(listPartsResponse.Parts, newPart)
+		listPartsResponse.Parts[index] = newPart
 	}
 	return listPartsResponse
 }
 
 // generateListMultipartUploadsResponse
-func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse {
+func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse {
 	listMultipartUploadsResponse := ListMultipartUploadsResponse{}
 	listMultipartUploadsResponse.Bucket = bucket
-	listMultipartUploadsResponse.Delimiter = metadata.Delimiter
-	listMultipartUploadsResponse.IsTruncated = metadata.IsTruncated
-	listMultipartUploadsResponse.EncodingType = metadata.EncodingType
-	listMultipartUploadsResponse.Prefix = metadata.Prefix
-	listMultipartUploadsResponse.KeyMarker = metadata.KeyMarker
-	listMultipartUploadsResponse.NextKeyMarker = metadata.NextKeyMarker
-	listMultipartUploadsResponse.MaxUploads = metadata.MaxUploads
-	listMultipartUploadsResponse.NextUploadIDMarker = metadata.NextUploadIDMarker
-	listMultipartUploadsResponse.UploadIDMarker = metadata.UploadIDMarker
-
-	listMultipartUploadsResponse.Uploads = make([]Upload, len(metadata.Upload))
-	for _, upload := range metadata.Upload {
+	listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter
+	listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated
+	listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType
+	listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix
+	listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker
+	listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker
+	listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads
+	listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker
+	listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker
+	listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes))
+	for index, commonPrefix := range multipartsInfo.CommonPrefixes {
+		listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{
+			Prefix: commonPrefix,
+		}
+	}
+	listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads))
+	for index, upload := range multipartsInfo.Uploads {
 		newUpload := Upload{}
 		newUpload.UploadID = upload.UploadID
 		newUpload.Key = upload.Object
-		newUpload.Initiated = upload.Initiated.Format(timeFormatAMZ)
-		listMultipartUploadsResponse.Uploads = append(listMultipartUploadsResponse.Uploads, newUpload)
+		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZ)
+		listMultipartUploadsResponse.Uploads[index] = newUpload
 	}
 	return listMultipartUploadsResponse
 }

api-router.go

@@ -0,0 +1,86 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import router "github.com/gorilla/mux"
+
+// objectAPIHandler implements and provides http handlers for S3 API.
+type objectAPIHandlers struct {
+	ObjectAPI ObjectLayer
+}
+
+// registerAPIRouter - registers S3 compatible APIs.
+func registerAPIRouter(mux *router.Router, api objectAPIHandlers) {
+	// API Router
+	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
+
+	// Bucket router
+	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+
+	/// Object operations
+
+	// HeadObject
+	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
+	// PutObjectPart
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	// ListObjectPxarts
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
+	// CompleteMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// NewMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
+	// AbortMultipartUpload
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// GetObject
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
+	// CopyObject
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/).*?").HandlerFunc(api.CopyObjectHandler)
+	// PutObject
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
+	// DeleteObject
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
+
+	/// Bucket operations
+
+	// GetBucketLocation
+	bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
+	// GetBucketPolicy
+	bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
+	// ListMultipartUploads
+	bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
+	// ListObjects
+	bucket.Methods("GET").HandlerFunc(api.ListObjectsHandler)
+	// PutBucketPolicy
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
+	// PutBucket
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
+	// HeadBucket
+	bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
+	// PostPolicy
+	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
+	// DeleteMultipleObjects
+	bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
+	// DeleteBucketPolicy
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
+	// DeleteBucket
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
+
+	/// Root operation
+
+	// ListBuckets
+	apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
+}

appveyor.yml

@@ -10,36 +10,20 @@ environment:
   GOPATH: c:\gopath
   GO_EXTLINK_ENABLED: 0
   GO15VENDOREXPERIMENT: 1
-# Remove this after fixing erasure code.
-  GODEBUG: cgocheck=0
 
 # scripts that run after cloning repository
 install:
   - '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64'
-  - curl -fsSL -o c:\go\bin\yasm.exe http://www.tortall.net/projects/yasm/releases/yasm-1.3.0-win64.exe
   - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
   - rd C:\Go /s /q
   - appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.windows-amd64.zip
   - 7z x go1.6.windows-amd64.zip -oC:\ >nul
-  - curl -fsSL -o mingw-w64.7z http://downloads.sourceforge.net/project/mingw-w64-dgn/mingw-w64/mingw-w64-bin-x86_64-20151206.7z
-  - 7z x -oC:\ mingw-w64.7z > NUL
-  - set PATH=C:\mingw64\bin;%PATH%
-  - x86_64-w64-mingw32-gcc --version
   - go version
   - go env
-  - git clone https://github.com/minio/isa-l
-  - cd isa-l
-  - make -f Makefile.unx arch=mingw
-  - mv include isa-l
-  - set CGO_CPPFLAGS=-Ic:/gopath/src/github.com/minio/minio/isa-l
-  - set CGO_LDFLAGS=c:/gopath/src/github.com/minio/minio/isa-l/isa-l.a
-  - set CC=x86_64-w64-mingw32-gcc
-  - set CXX=x86_64-w64-mingw32-g++
   - cd %GOPATH%\src\github.com\minio\minio
 
 # to run your custom scripts instead of automatic MSBuild
 build_script:
-  - go generate github.com/minio/minio/pkg/crypto/sha1
   - go test .
   - go test -race .
   - go test github.com/minio/minio/pkg...

auth-handler.go

@@ -17,17 +17,14 @@
 package main
 
 import (
+	"bytes"
+	"crypto/md5"
 	"crypto/sha256"
+	"encoding/base64"
 	"encoding/hex"
+	"io/ioutil"
 	"net/http"
 	"strings"
-
-	"github.com/minio/minio/pkg/s3/signature4"
-)
-
-const (
-	signV4Algorithm = "AWS4-HMAC-SHA256"
-	jwtAlgorithm    = "Bearer"
 )
 
 // Verify if request has JWT.
@@ -68,14 +65,6 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool {
 	return false
 }
 
-// Verify if incoming request is anonymous.
-func isRequestAnonymous(r *http.Request) bool {
-	if isRequestJWT(r) || isRequestSignatureV4(r) || isRequestPresignedSignatureV4(r) || isRequestPostPolicySignatureV4(r) {
-		return false
-	}
-	return true
-}
-
 // Authorization type.
 type authType int
 
@@ -105,30 +94,42 @@ func getRequestAuthType(r *http.Request) authType {
 	return authTypeUnknown
 }
 
+// sum256 calculate sha256 sum for an input byte array
+func sum256(data []byte) []byte {
+	hash := sha256.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// sumMD5 calculate md5 sum for an input byte array
+func sumMD5(data []byte) []byte {
+	hash := md5.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
 // Verify if request has valid AWS Signature Version '4'.
-func isReqAuthenticated(sign *signature4.Sign, r *http.Request) (s3Error APIErrorCode) {
-	auth := sign.SetHTTPRequestToVerify(r)
+func isReqAuthenticated(r *http.Request) (s3Error APIErrorCode) {
+	if r == nil {
+		return ErrInternalError
+	}
+	payload, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return ErrInternalError
+	}
+	// Verify Content-Md5, if payload is set.
+	if r.Header.Get("Content-Md5") != "" {
+		if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+			return ErrBadDigest
+		}
+	}
+	// Populate back the payload.
+	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+	validateRegion := true // Validate region.
 	if isRequestSignatureV4(r) {
-		dummyPayload := sha256.Sum256([]byte(""))
-		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(dummyPayload[:]))
-		if err != nil {
-			errorIf(err.Trace(), "Signature verification failed.", nil)
-			return ErrInternalError
-		}
-		if !ok {
-			return ErrSignatureDoesNotMatch
-		}
-		return ErrNone
+		return doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
 	} else if isRequestPresignedSignatureV4(r) {
-		ok, err := auth.DoesPresignedSignatureMatch()
-		if err != nil {
-			errorIf(err.Trace(), "Presigned signature verification failed.", nil)
-			return ErrInternalError
-		}
-		if !ok {
-			return ErrSignatureDoesNotMatch
-		}
-		return ErrNone
+		return doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
 	}
 	return ErrAccessDenied
 }

bucket-handlers.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"bytes"
-	"crypto/md5"
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/xml"
@@ -30,23 +29,18 @@ import (
 	"strings"
 
 	mux "github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/crypto/sha256"
-	"github.com/minio/minio/pkg/fs"
-	"github.com/minio/minio/pkg/probe"
-	"github.com/minio/minio/pkg/s3/access"
-	"github.com/minio/minio/pkg/s3/signature4"
 )
 
-// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) {
 	// Read saved bucket policy.
 	policy, err := readBucketPolicy(bucket)
 	if err != nil {
-		errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
+		errorIf(err, "Unable read bucket policy.")
+		switch err.(type) {
+		case BucketNotFound:
 			return ErrNoSuchBucket
-		case fs.BucketNameInvalid:
+		case BucketNameInvalid:
 			return ErrInvalidBucketName
 		default:
 			// For any other error just return AccessDenied.
@@ -54,23 +48,23 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error
 		}
 	}
 	// Parse the saved policy.
-	accessPolicy, e := accesspolicy.Validate(policy)
-	if e != nil {
-		errorIf(probe.NewError(e), "Parse policy failed.", nil)
+	bucketPolicy, err := parseBucketPolicy(policy)
+	if err != nil {
+		errorIf(err, "Unable to parse bucket policy.")
 		return ErrAccessDenied
 	}
 
-	// Construct resource in 'arn:aws:s3:::examplebucket' format.
-	resource := accesspolicy.AWSResourcePrefix + strings.TrimPrefix(reqURL.Path, "/")
+	// Construct resource in 'arn:aws:s3:::examplebucket/object' format.
+	resource := AWSResourcePrefix + strings.TrimPrefix(reqURL.Path, "/")
 
 	// Get conditions for policy verification.
 	conditions := make(map[string]string)
 	for queryParam := range reqURL.Query() {
-		conditions[queryParam] = reqURL.Query().Get("queryParam")
+		conditions[queryParam] = reqURL.Query().Get(queryParam)
 	}
 
 	// Validate action, resource and conditions with current policy statements.
-	if !bucketPolicyEvalStatements(action, resource, conditions, accessPolicy.Statements) {
+	if !bucketPolicyEvalStatements(action, resource, conditions, bucketPolicy.Statements) {
 		return ErrAccessDenied
 	}
 	return ErrNone
@@ -79,7 +73,7 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error
 // GetBucketLocationHandler - GET Bucket location.
 // -------------------------
 // This operation returns bucket location.
-func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -89,40 +83,55 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
 	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 		if s3Error := enforceBucketPolicy("s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	case authTypeSigned, authTypePresigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		payload, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		// Verify Content-Md5, if payload is set.
+		if r.Header.Get("Content-Md5") != "" {
+			if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+				writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
+				return
+			}
+		}
+		// Populate back the payload.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+		var s3Error APIErrorCode // API error code.
+		validateRegion := false  // Validate region.
+		if isRequestSignatureV4(r) {
+			s3Error = doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		} else if isRequestPresignedSignatureV4(r) {
+			s3Error = doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		}
+		if s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	_, err := api.Filesystem.GetBucketMetadata(bucket)
-	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		default:
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-		}
+	if _, err := api.ObjectAPI.GetBucketInfo(bucket); err != nil {
+		errorIf(err, "Unable to fetch bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 		return
 	}
 
 	// Generate response.
 	encodedSuccessResponse := encodeResponse(LocationResponse{})
-	if api.Region != "us-east-1" {
+	// Get current region.
+	region := serverConfig.GetRegion()
+	if region != "us-east-1" {
 		encodedSuccessResponse = encodeResponse(LocationResponse{
-			Location: api.Region,
+			Location: region,
 		})
 	}
-	setCommonHeaders(w) // write headers.
+	setCommonHeaders(w) // Write headers.
 	writeSuccessResponse(w, encodedSuccessResponse)
 }
 
@@ -134,7 +143,7 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re
 // completed or aborted. This operation returns at most 1,000 multipart
 // uploads in the response.
 //
-func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -150,34 +159,33 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http
 			return
 		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	resources := getBucketMultipartResources(r.URL.Query())
-	if resources.MaxUploads < 0 {
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
+	if maxUploads < 0 {
 		writeErrorResponse(w, r, ErrInvalidMaxUploads, r.URL.Path)
 		return
 	}
-	if resources.MaxUploads == 0 {
-		resources.MaxUploads = maxObjectList
+	if keyMarker != "" {
+		// Marker not common with prefix is not implemented.
+		if !strings.HasPrefix(keyMarker, prefix) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
 	}
 
-	resources, err := api.Filesystem.ListMultipartUploads(bucket, resources)
+	listMultipartsInfo, err := api.ObjectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
-		errorIf(err.Trace(), "ListMultipartUploads failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-		default:
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-		}
+		errorIf(err, "Unable to list multipart uploads.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 		return
 	}
 	// generate response
-	response := generateListMultipartUploadsResponse(bucket, resources)
+	response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo)
 	encodedSuccessResponse := encodeResponse(response)
 	// write headers.
 	setCommonHeaders(w)
@@ -191,7 +199,7 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
-func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -201,59 +209,78 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request)
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
 	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	case authTypeSigned, authTypePresigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
-
+	var prefix, marker, token, delimiter, startAfter string
+	var maxkeys int
+	var listV2 bool
 	// TODO handle encoding type.
-	prefix, marker, delimiter, maxkeys, _ := getBucketResources(r.URL.Query())
+	if r.URL.Query().Get("list-type") == "2" {
+		listV2 = true
+		prefix, token, startAfter, delimiter, maxkeys, _ = getListObjectsV2Args(r.URL.Query())
+		// For ListV2 "start-after" is considered only if "continuation-token" is empty.
+		if token == "" {
+			marker = startAfter
+		} else {
+			marker = token
+		}
+	} else {
+		prefix, marker, delimiter, maxkeys, _ = getListObjectsV1Args(r.URL.Query())
+	}
 	if maxkeys < 0 {
 		writeErrorResponse(w, r, ErrInvalidMaxKeys, r.URL.Path)
 		return
 	}
-	if maxkeys == 0 {
-		maxkeys = maxObjectList
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+		return
+	}
+	// If marker is set unescape.
+	if marker != "" {
+		// Marker not common with prefix is not implemented.
+		if !strings.HasPrefix(marker, prefix) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
 	}
 
-	listResp, err := api.Filesystem.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
+	listObjectsInfo, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
+
 	if err == nil {
+		var encodedSuccessResponse []byte
 		// generate response
-		response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listResp)
-		encodedSuccessResponse := encodeResponse(response)
+		if listV2 {
+			response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, maxkeys, listObjectsInfo)
+			encodedSuccessResponse = encodeResponse(response)
+		} else {
+			response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listObjectsInfo)
+			encodedSuccessResponse = encodeResponse(response)
+		}
 		// Write headers
 		setCommonHeaders(w)
 		// Write success response.
 		writeSuccessResponse(w, encodedSuccessResponse)
 		return
 	}
-	switch err.ToGoError().(type) {
-	case fs.BucketNameInvalid:
-		writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-	case fs.BucketNotFound:
-		writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-	case fs.ObjectNotFound:
-		writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
-	case fs.ObjectNameInvalid:
-		writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
-	default:
-		errorIf(err.Trace(), "ListObjects failed.", nil)
-		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-	}
+	errorIf(err, "Unable to list objects.")
+	writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 }
 
 // ListBucketsHandler - GET Service
 // -----------
 // This implementation of the GET operation returns a list of all buckets
 // owned by the authenticated sender of the request.
-func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	// List buckets does not support bucket policies.
 	switch getRequestAuthType(r) {
 	default:
@@ -261,16 +288,37 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request)
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
 	case authTypeSigned, authTypePresigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		payload, e := ioutil.ReadAll(r.Body)
+		if e != nil {
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		// Verify Content-Md5, if payload is set.
+		if r.Header.Get("Content-Md5") != "" {
+			if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+				writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
+				return
+			}
+		}
+		// Populate back the payload.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+		var s3Error APIErrorCode // API error code.
+		validateRegion := false  // Validate region.
+		if isRequestSignatureV4(r) {
+			s3Error = doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		} else if isRequestPresignedSignatureV4(r) {
+			s3Error = doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		}
+		if s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	buckets, err := api.Filesystem.ListBuckets()
+	bucketsInfo, err := api.ObjectAPI.ListBuckets()
 	if err == nil {
 		// generate response
-		response := generateListBucketsResponse(buckets)
+		response := generateListBucketsResponse(bucketsInfo)
 		encodedSuccessResponse := encodeResponse(response)
 		// write headers
 		setCommonHeaders(w)
@@ -278,15 +326,33 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request)
 		writeSuccessResponse(w, encodedSuccessResponse)
 		return
 	}
-	errorIf(err.Trace(), "ListBuckets failed.", nil)
-	writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+	errorIf(err, "Unable to list buckets.")
+	writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 }
 
 // DeleteMultipleObjectsHandler - deletes multiple objects.
-func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
@@ -301,69 +367,20 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Set http request for signature.
-	auth := api.Signature.SetHTTPRequestToVerify(r)
-
 	// Allocate incoming content length bytes.
 	deleteXMLBytes := make([]byte, r.ContentLength)
 
 	// Read incoming body XML bytes.
-	_, e := io.ReadFull(r.Body, deleteXMLBytes)
-	if e != nil {
-		errorIf(probe.NewError(e), "DeleteMultipleObjects failed.", nil)
+	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
+		errorIf(err, "Unable to read HTTP body.")
 		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		return
 	}
 
-	switch getRequestAuthType(r) {
-	default:
-		// For all unknown auth types return error.
-		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
-		return
-	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
-			writeErrorResponse(w, r, s3Error, r.URL.Path)
-			return
-		}
-	case authTypePresigned:
-		// Check if request is presigned.
-		ok, err := auth.DoesPresignedSignatureMatch()
-		if err != nil {
-			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-	case authTypeSigned:
-		// Check if request is signed.
-		sha := sha256.New()
-		mdSh := md5.New()
-		sha.Write(deleteXMLBytes)
-		mdSh.Write(deleteXMLBytes)
-		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sha.Sum(nil)))
-		if err != nil {
-			errorIf(err.Trace(), "DeleteMultipleObjects failed.", nil)
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		// Verify content md5.
-		if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(mdSh.Sum(nil)) {
-			writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
-			return
-		}
-	}
-
 	// Unmarshal list of keys to be deleted.
 	deleteObjects := &DeleteObjectsRequest{}
-	if e := xml.Unmarshal(deleteXMLBytes, deleteObjects); e != nil {
+	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
+		errorIf(err, "Unable to unmarshal delete objects request XML.")
 		writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
 		return
 	}
@@ -372,45 +389,18 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
 	var deletedObjects []ObjectIdentifier
 	// Loop through all the objects and delete them sequentially.
 	for _, object := range deleteObjects.Objects {
-		err := api.Filesystem.DeleteObject(bucket, object.ObjectName)
+		err := api.ObjectAPI.DeleteObject(bucket, object.ObjectName)
 		if err == nil {
 			deletedObjects = append(deletedObjects, ObjectIdentifier{
 				ObjectName: object.ObjectName,
 			})
 		} else {
-			errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil)
-			switch err.ToGoError().(type) {
-			case fs.BucketNameInvalid:
-				deleteErrors = append(deleteErrors, DeleteError{
-					Code:    errorCodeResponse[ErrInvalidBucketName].Code,
-					Message: errorCodeResponse[ErrInvalidBucketName].Description,
-					Key:     object.ObjectName,
-				})
-			case fs.BucketNotFound:
-				deleteErrors = append(deleteErrors, DeleteError{
-					Code:    errorCodeResponse[ErrNoSuchBucket].Code,
-					Message: errorCodeResponse[ErrNoSuchBucket].Description,
-					Key:     object.ObjectName,
-				})
-			case fs.ObjectNotFound:
-				deleteErrors = append(deleteErrors, DeleteError{
-					Code:    errorCodeResponse[ErrNoSuchKey].Code,
-					Message: errorCodeResponse[ErrNoSuchKey].Description,
-					Key:     object.ObjectName,
-				})
-			case fs.ObjectNameInvalid:
-				deleteErrors = append(deleteErrors, DeleteError{
-					Code:    errorCodeResponse[ErrNoSuchKey].Code,
-					Message: errorCodeResponse[ErrNoSuchKey].Description,
-					Key:     object.ObjectName,
-				})
-			default:
-				deleteErrors = append(deleteErrors, DeleteError{
-					Code:    errorCodeResponse[ErrInternalError].Code,
-					Message: errorCodeResponse[ErrInternalError].Description,
-					Key:     object.ObjectName,
-				})
-			}
+			errorIf(err, "Unable to delete object.")
+			deleteErrors = append(deleteErrors, DeleteError{
+				Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
+				Message: errorCodeResponse[toAPIErrorCode(err)].Description,
+				Key:     object.ObjectName,
+			})
 		}
 	}
 	// Generate response
@@ -425,68 +415,36 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
 // PutBucketHandler - PUT Bucket
 // ----------
 // This implementation of the PUT operation creates a new bucket for authenticated request
-func (api storageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	// Set http request for signature.
-	auth := api.Signature.SetHTTPRequestToVerify(r)
 	switch getRequestAuthType(r) {
 	default:
 		// For all unknown auth types return error.
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
-	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:CreateBucket", bucket, r.URL); s3Error != ErrNone {
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
-	case authTypePresigned:
-		ok, err := auth.DoesPresignedSignatureMatch()
-		if err != nil {
-			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-	case authTypeSigned:
-		// Verify signature for the incoming body if any.
-		locationBytes, e := ioutil.ReadAll(r.Body)
-		if e != nil {
-			errorIf(probe.NewError(e), "MakeBucket failed.", nil)
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-			return
-		}
-		sh := sha256.New()
-		sh.Write(locationBytes)
-		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
-		if err != nil {
-			errorIf(err.Trace(), "MakeBucket failed.", nil)
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
 	}
 
+	// the location value in the request body should match the Region in serverConfig.
+	// other values of location are not accepted.
+	// make bucket fails in such cases.
+	errCode := isValidLocationContraint(r.Body, serverConfig.GetRegion())
+	if errCode != ErrNone {
+		writeErrorResponse(w, r, errCode, r.URL.Path)
+		return
+	}
 	// Make bucket.
-	err := api.Filesystem.MakeBucket(bucket)
+	err := api.ObjectAPI.MakeBucket(bucket)
 	if err != nil {
-		errorIf(err.Trace(), "MakeBucket failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		case fs.BucketExists:
-			writeErrorResponse(w, r, ErrBucketAlreadyExists, r.URL.Path)
-		default:
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-		}
+		errorIf(err, "Unable to create a bucket.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 		return
 	}
 	// Make sure to add Location information here only for bucket
@@ -494,24 +452,25 @@ func (api storageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponse(w, nil)
 }
 
-func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]string, *probe.Error) {
+func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]string, error) {
 	/// HTML Form values
 	formValues := make(map[string]string)
 	filePart := new(bytes.Buffer)
-	var e error
-	for e == nil {
+	var err error
+	for err == nil {
 		var part *multipart.Part
-		part, e = reader.NextPart()
+		part, err = reader.NextPart()
 		if part != nil {
 			if part.FileName() == "" {
-				buffer, e := ioutil.ReadAll(part)
-				if e != nil {
-					return nil, nil, probe.NewError(e)
+				var buffer []byte
+				buffer, err = ioutil.ReadAll(part)
+				if err != nil {
+					return nil, nil, err
 				}
 				formValues[http.CanonicalHeaderKey(part.FormName())] = string(buffer)
 			} else {
-				if _, e := io.Copy(filePart, part); e != nil {
-					return nil, nil, probe.NewError(e)
+				if _, err = io.Copy(filePart, part); err != nil {
+					return nil, nil, err
 				}
 			}
 		}
@@ -523,72 +482,58 @@ func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]stri
 // ----------
 // This implementation of the POST operation handles object creation with a specified
 // signature policy in multipart/form-data
-func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	// Here the parameter is the size of the form data that should
-	// be loaded in memory, the remaining being put in temporary
-	// files
-	reader, e := r.MultipartReader()
-	if e != nil {
-		errorIf(probe.NewError(e), "Unable to initialize multipart reader.", nil)
+	// be loaded in memory, the remaining being put in temporary files.
+	reader, err := r.MultipartReader()
+	if err != nil {
+		errorIf(err, "Unable to initialize multipart reader.")
 		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
 		return
 	}
 
 	fileBody, formValues, err := extractHTTPFormValues(reader)
 	if err != nil {
-		errorIf(err.Trace(), "Unable to parse form values.", nil)
+		errorIf(err, "Unable to parse form values.")
 		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
 		return
 	}
 	bucket := mux.Vars(r)["bucket"]
 	formValues["Bucket"] = bucket
 	object := formValues["Key"]
-	var ok bool
-
-	// Set http request for signature.
-	auth := api.Signature.SetHTTPRequestToVerify(r)
 
 	// Verify policy signature.
-	ok, err = auth.DoesPolicySignatureMatch(formValues)
+	apiErr := doesPolicySignatureMatch(formValues)
+	if apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+	if apiErr = checkPostPolicy(formValues); apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+
+	// Save metadata.
+	metadata := make(map[string]string)
+	// Nothing to store right now.
+
+	md5Sum, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, metadata)
 	if err != nil {
-		errorIf(err.Trace(), "Unable to verify signature.", nil)
-		writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+		errorIf(err, "Unable to create object.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 		return
 	}
-	if !ok {
-		writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-		return
+	if md5Sum != "" {
+		w.Header().Set("ETag", "\""+md5Sum+"\"")
 	}
-	if err = signature4.ApplyPolicyCond(formValues); err != nil {
-		errorIf(err.Trace(), "Invalid request, policy doesn't match with the endpoint.", nil)
-		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
-		return
-	}
-	metadata, err := api.Filesystem.CreateObject(bucket, object, "", -1, fileBody, nil)
-	if err != nil {
-		errorIf(err.Trace(), "CreateObject failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.RootPathFull:
-			writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
-		case fs.BucketNotFound:
-			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		case fs.BadDigest:
-			writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
-		case fs.IncompleteBody:
-			writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path)
-		case fs.InvalidDigest:
-			writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path)
-		default:
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-		}
-		return
-	}
-	if metadata.MD5 != "" {
-		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
-	}
-	writeSuccessResponse(w, nil)
+	encodedSuccessResponse := encodeResponse(PostResponse{
+		Location: getObjectLocation(bucket, object), // TODO Full URL is preferred
+		Bucket:   bucket,
+		Key:      object,
+		ETag:     md5Sum,
+	})
+	setCommonHeaders(w)
+	writeSuccessResponse(w, encodedSuccessResponse)
 }
 
 // HeadBucketHandler - HEAD Bucket
@@ -597,40 +542,7 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req
 // The operation returns a 200 OK if the bucket exists and you
 // have permission to access it. Otherwise, the operation might
 // return responses such as 404 Not Found and 403 Forbidden.
-func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	switch getRequestAuthType(r) {
-	default:
-		// For all unknown auth types return error.
-		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
-		return
-	case authTypePresigned, authTypeSigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
-			writeErrorResponse(w, r, s3Error, r.URL.Path)
-			return
-		}
-	}
-
-	_, err := api.Filesystem.GetBucketMetadata(bucket)
-	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		default:
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-		}
-		return
-	}
-	writeSuccessResponse(w, nil)
-}
-
-// DeleteBucketHandler - Delete bucket
-func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -640,29 +552,46 @@ func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
 	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:DeleteBucket", bucket, r.URL); s3Error != ErrNone {
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	err := api.Filesystem.DeleteBucket(bucket)
-	if err != nil {
-		errorIf(err.Trace(), "DeleteBucket failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-		case fs.BucketNotEmpty:
-			writeErrorResponse(w, r, ErrBucketNotEmpty, r.URL.Path)
-		default:
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+	if _, err := api.ObjectAPI.GetBucketInfo(bucket); err != nil {
+		errorIf(err, "Unable to fetch bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// DeleteBucketHandler - Delete bucket
+func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
 		}
+	}
+
+	if err := api.ObjectAPI.DeleteBucket(bucket); err != nil {
+		errorIf(err, "Unable to delete a bucket.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 		return
 	}
 

bucket-policy-handlers.go

@@ -18,8 +18,6 @@ package main
 
 import (
 	"bytes"
-	"crypto/sha256"
-	"encoding/hex"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -27,9 +25,6 @@ import (
 	"strings"
 
 	mux "github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/fs"
-	"github.com/minio/minio/pkg/probe"
-	"github.com/minio/minio/pkg/s3/access"
 )
 
 // maximum supported access policy size.
@@ -37,12 +32,13 @@ const maxAccessPolicySize = 20 * 1024 * 1024 // 20KiB.
 
 // Verify if a given action is valid for the url path based on the
 // existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions map[string]string, statements []accesspolicy.Statement) bool {
+func bucketPolicyEvalStatements(action string, resource string, conditions map[string]string, statements []policyStatement) bool {
 	for _, statement := range statements {
 		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
 			if statement.Effect == "Allow" {
 				return true
 			}
+			// Do not uncomment kept here for readability.
 			// else statement.Effect == "Deny"
 			return false
 		}
@@ -52,7 +48,7 @@ func bucketPolicyEvalStatements(action string, resource string, conditions map[s
 }
 
 // Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions map[string]string, statement accesspolicy.Statement) bool {
+func bucketPolicyMatchStatement(action string, resource string, conditions map[string]string, statement policyStatement) bool {
 	// Verify if action matches.
 	if bucketPolicyActionMatch(action, statement) {
 		// Verify if resource matches.
@@ -67,11 +63,11 @@ func bucketPolicyMatchStatement(action string, resource string, conditions map[s
 }
 
 // Verify if given action matches with policy statement.
-func bucketPolicyActionMatch(action string, statement accesspolicy.Statement) bool {
+func bucketPolicyActionMatch(action string, statement policyStatement) bool {
 	for _, policyAction := range statement.Actions {
 		// Policy action can be a regex, validate the action with matching string.
-		matched, e := regexp.MatchString(policyAction, action)
-		fatalIf(probe.NewError(e), "Invalid pattern, please verify the pattern string.", nil)
+		matched, err := regexp.MatchString(policyAction, action)
+		fatalIf(err, "Invalid action \"%s\" in bucket policy.", action)
 		if matched {
 			return true
 		}
@@ -79,21 +75,49 @@ func bucketPolicyActionMatch(action string, statement accesspolicy.Statement) bo
 	return false
 }
 
+// Match function matches wild cards in 'pattern' for resource.
+func resourceMatch(pattern, resource string) bool {
+	if pattern == "" {
+		return resource == pattern
+	}
+	if pattern == "*" {
+		return true
+	}
+	parts := strings.Split(pattern, "*")
+	if len(parts) == 1 {
+		return resource == pattern
+	}
+	tGlob := strings.HasSuffix(pattern, "*")
+	end := len(parts) - 1
+	if !strings.HasPrefix(resource, parts[0]) {
+		return false
+	}
+	for i := 1; i < end; i++ {
+		if !strings.Contains(resource, parts[i]) {
+			return false
+		}
+		idx := strings.Index(resource, parts[i]) + len(parts[i])
+		resource = resource[idx:]
+	}
+	return tGlob || strings.HasSuffix(resource, parts[end])
+}
+
 // Verify if given resource matches with policy statement.
-func bucketPolicyResourceMatch(resource string, statement accesspolicy.Statement) bool {
-	for _, presource := range statement.Resources {
-		matched, e := regexp.MatchString(presource, strings.TrimPrefix(resource, "/"))
-		fatalIf(probe.NewError(e), "Invalid pattern, please verify the pattern string.", nil)
-		// For any path matches, we return quickly and the let the caller continue.
-		if matched {
-			return true
+func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
+	for _, resourcep := range statement.Resources {
+		// the resource rule for object could contain "*" wild card.
+		// the requested object can be given access based on the already set bucket policy if
+		// the match is successful.
+		// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html .
+		if matched := resourceMatch(resourcep, resource); !matched {
+			return false
 		}
 	}
-	return false
+	return true
 }
 
 // Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions map[string]string, statement accesspolicy.Statement) bool {
+func bucketPolicyConditionMatch(conditions map[string]string, statement policyStatement) bool {
 	// Supports following conditions.
 	// - StringEquals
 	// - StringNotEquals
@@ -130,10 +154,22 @@ func bucketPolicyConditionMatch(conditions map[string]string, statement accesspo
 // -----------------
 // This implementation of the PUT operation uses the policy
 // subresource to add to or replace a policy on a bucket
-func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
 	// If Content-Length is unknown or zero, deny the
 	// request. PutBucketPolicy always needs a Content-Length if
 	// incoming request is not chunked.
@@ -152,65 +188,32 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ
 	// Read access policy up to maxAccessPolicySize.
 	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 	// bucket policies are limited to 20KB in size, using a limit reader.
-	accessPolicyBytes, e := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
-	if e != nil {
-		errorIf(probe.NewError(e).Trace(bucket), "Reading policy failed.", nil)
+	bucketPolicyBuf, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	if err != nil {
+		errorIf(err, "Unable to read bucket policy.")
 		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		return
 	}
 
-	// Parse access access.
-	accessPolicy, e := accesspolicy.Validate(accessPolicyBytes)
-	if e != nil {
+	// Parse bucket policy.
+	bucketPolicy, err := parseBucketPolicy(bucketPolicyBuf)
+	if err != nil {
+		errorIf(err, "Unable to parse bucket policy.")
 		writeErrorResponse(w, r, ErrInvalidPolicyDocument, r.URL.Path)
 		return
 	}
 
-	// If the policy resource has different bucket name, reject it.
-	for _, statement := range accessPolicy.Statements {
-		for _, resource := range statement.Resources {
-			resourcePrefix := strings.SplitAfter(resource, accesspolicy.AWSResourcePrefix)[1]
-			if !strings.HasPrefix(resourcePrefix, bucket) {
-				writeErrorResponse(w, r, ErrMalformedPolicy, r.URL.Path)
-				return
-			}
-		}
-	}
-
-	// Set http request for signature verification.
-	auth := api.Signature.SetHTTPRequestToVerify(r)
-	if isRequestPresignedSignatureV4(r) {
-		ok, err := auth.DoesPresignedSignatureMatch()
-		if err != nil {
-			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-	} else if isRequestSignatureV4(r) {
-		sh := sha256.New()
-		sh.Write(accessPolicyBytes)
-		ok, err := api.Signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
-		if err != nil {
-			errorIf(err.Trace(string(accessPolicyBytes)), "SaveBucketPolicy failed.", nil)
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
+	// Parse check bucket policy.
+	if s3Error := checkBucketPolicyResources(bucket, bucketPolicy); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
 	}
 
 	// Save bucket policy.
-	err := writeBucketPolicy(bucket, accessPolicyBytes)
-	if err != nil {
-		errorIf(err.Trace(bucket), "SaveBucketPolicy failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
+	if err := writeBucketPolicy(bucket, bucketPolicyBuf); err != nil {
+		errorIf(err, "Unable to write bucket policy.")
+		switch err.(type) {
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@@ -224,24 +227,29 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ
 // -----------------
 // This implementation of the DELETE operation uses the policy
 // subresource to add to remove a policy on a bucket.
-func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	// Validate incoming signature.
-	if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
-		writeErrorResponse(w, r, s3Error, r.URL.Path)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
 	}
 
 	// Delete bucket access policy.
-	err := removeBucketPolicy(bucket)
-	if err != nil {
-		errorIf(err.Trace(bucket), "DeleteBucketPolicy failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
+	if err := removeBucketPolicy(bucket); err != nil {
+		errorIf(err, "Unable to remove bucket policy.")
+		switch err.(type) {
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		case fs.BucketPolicyNotFound:
+		case BucketPolicyNotFound:
 			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@@ -255,24 +263,30 @@ func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.R
 // -----------------
 // This operation uses the policy
 // subresource to return the policy of a specified bucket.
-func (api storageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	// Validate incoming signature.
-	if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
-		writeErrorResponse(w, r, s3Error, r.URL.Path)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
 	}
 
 	// Read bucket access policy.
 	p, err := readBucketPolicy(bucket)
 	if err != nil {
-		errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
+		errorIf(err, "Unable to read bucket policy.")
+		switch err.(type) {
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		case fs.BucketPolicyNotFound:
+		case BucketPolicyNotFound:
 			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)

bucket-policy-handlers_test.go

@@ -0,0 +1,78 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+// Tests validate Bucket policy resource matcher.
+func TestBucketPolicyResourceMatch(t *testing.T) {
+
+	// generates\ statement with given resource..
+	generateStatement := func(resource string) policyStatement {
+		statement := policyStatement{}
+		statement.Resources = []string{resource}
+		return statement
+	}
+
+	// generates resource prefix.
+	generateResource := func(bucketName, objectName string) string {
+		return AWSResourcePrefix + bucketName + "/" + objectName
+	}
+
+	testCases := []struct {
+		resourceToMatch       string
+		statement             policyStatement
+		expectedResourceMatch bool
+	}{
+		// Test case 1-4.
+		// Policy with resource ending with bucket/* allows access to all objects inside the given bucket.
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		// Test case - 5.
+		// Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt.
+		{generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), false},
+		// Test case - 6.
+		// Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt.
+		{generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true},
+		// Test case - 7.
+		// Policy with resource ending with bucket/oo* allows access to all subfolders starting with "oo" inside given bucket.
+		{generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true},
+		// Test case - 8.
+		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false},
+		// Test case - 9.
+		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false},
+		// Test case - 10.
+		// Proves that the name space is flat.
+		{generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix,
+			"minio-bucket"+"/*/India/*/Bihar")), true},
+		// Test case - 11.
+		// Proves that the name space is flat.
+		{generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix,
+			"minio-bucket"+"/*/India/*/Bihar/*")), true},
+	}
+	for i, testCase := range testCases {
+		actualResourceMatch := bucketPolicyResourceMatch(testCase.resourceToMatch, testCase.statement)
+		if testCase.expectedResourceMatch != actualResourceMatch {
+			t.Errorf("Test %d: Expected Resource match to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResourceMatch, actualResourceMatch)
+		}
+	}
+}

bucket-policy-parser.go

@@ -14,14 +14,16 @@
  * limitations under the License.
  */
 
-// Package accesspolicy implements AWS Access Policy Language parser in
+// This file implements AWS Access Policy Language parser in
 // accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-package accesspolicy
+package main
 
 import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"path"
+	"sort"
 	"strings"
 )
 
@@ -32,30 +34,39 @@ const (
 
 // supportedActionMap - lists all the actions supported by minio.
 var supportedActionMap = map[string]struct{}{
-	"*":                             {},
-	"s3:*":                          {},
 	"s3:GetObject":                  {},
 	"s3:ListBucket":                 {},
 	"s3:PutObject":                  {},
-	"s3:CreateBucket":               {},
 	"s3:GetBucketLocation":          {},
-	"s3:DeleteBucket":               {},
 	"s3:DeleteObject":               {},
 	"s3:AbortMultipartUpload":       {},
 	"s3:ListBucketMultipartUploads": {},
 	"s3:ListMultipartUploadParts":   {},
 }
 
+// supported Conditions type.
+var supportedConditionsType = map[string]struct{}{
+	"StringEquals":    {},
+	"StringNotEquals": {},
+}
+
+// Validate s3:prefix, s3:max-keys are present if not
+// supported keys for the conditions.
+var supportedConditionsKey = map[string]struct{}{
+	"s3:prefix":   {},
+	"s3:max-keys": {},
+}
+
 // User - canonical users list.
-type User struct {
+type policyUser struct {
 	AWS []string
 }
 
 // Statement - minio policy statement
-type Statement struct {
+type policyStatement struct {
 	Sid        string
 	Effect     string
-	Principal  User
+	Principal  policyUser                   `json:"Principal"`
 	Actions    []string                     `json:"Action"`
 	Resources  []string                     `json:"Resource"`
 	Conditions map[string]map[string]string `json:"Condition"`
@@ -63,8 +74,8 @@ type Statement struct {
 
 // BucketPolicy - minio policy collection
 type BucketPolicy struct {
-	Version    string      // date in 0000-00-00 format
-	Statements []Statement `json:"Statement"`
+	Version    string            // date in 0000-00-00 format
+	Statements []policyStatement `json:"Statement"`
 }
 
 // supportedEffectMap - supported effects.
@@ -132,60 +143,130 @@ func isValidPrincipals(principals []string) (err error) {
 		err = errors.New("Principal cannot be empty.")
 		return err
 	}
-	var ok bool
 	for _, principal := range principals {
 		// Minio does not support or implement IAM, "*" is the only valid value.
-		if principal == "*" {
-			ok = true
-			continue
-		}
-		ok = false
-	}
-	if !ok {
-		err = errors.New("Unsupported principal style found: ‘" + strings.Join(principals, " ") + "’, please validate your policy document.")
-		return err
-	}
-	return nil
-}
-
-func isValidConditions(conditions map[string]map[string]string) (err error) {
-	// Verify conditions should be valid.
-	if len(conditions) > 0 {
-		// Validate if stringEquals, stringNotEquals are present
-		// if not throw an error.
-		_, stringEqualsOK := conditions["StringEquals"]
-		_, stringNotEqualsOK := conditions["StringNotEquals"]
-		if !stringEqualsOK && !stringNotEqualsOK {
-			err = fmt.Errorf("Unsupported condition type found: ‘%s’, please validate your policy document.", conditions)
+		// Amazon s3 doc on principals: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
+		if principal != "*" {
+			err = fmt.Errorf("Unsupported principal style found: ‘%s’, please validate your policy document.", principal)
 			return err
 		}
-		// Validate s3:prefix, s3:max-keys are present if not
-		// throw an error.
-		if len(conditions["StringEquals"]) > 0 {
-			_, s3PrefixOK := conditions["StringEquals"]["s3:prefix"]
-			_, s3MaxKeysOK := conditions["StringEquals"]["s3:max-keys"]
-			if !s3PrefixOK && !s3MaxKeysOK {
-				err = fmt.Errorf("Unsupported condition keys found: ‘%s’, please validate your policy document.",
-					conditions["StringEquals"])
-				return err
+	}
+	return nil
+}
+
+// isValidConditions - are valid conditions.
+func isValidConditions(conditions map[string]map[string]string) (err error) {
+	// Returns true if string 'a' is found in the list.
+	findString := func(a string, list []string) bool {
+		for _, b := range list {
+			if b == a {
+				return true
 			}
 		}
-		if len(conditions["StringNotEquals"]) > 0 {
-			_, s3PrefixOK := conditions["StringNotEquals"]["s3:prefix"]
-			_, s3MaxKeysOK := conditions["StringNotEquals"]["s3:max-keys"]
-			if !s3PrefixOK && !s3MaxKeysOK {
-				err = fmt.Errorf("Unsupported condition keys found: ‘%s’, please validate your policy document.",
-					conditions["StringNotEquals"])
+		return false
+	}
+	conditionKeyVal := make(map[string][]string)
+	// Verify conditions should be valid.
+	// Validate if stringEquals, stringNotEquals are present
+	// if not throw an error.
+	for conditionType := range conditions {
+		_, validType := supportedConditionsType[conditionType]
+		if !validType {
+			err = fmt.Errorf("Unsupported condition type '%s', please validate your policy document.", conditionType)
+			return err
+		}
+		for key := range conditions[conditionType] {
+			_, validKey := supportedConditionsKey[key]
+			if !validKey {
+				err = fmt.Errorf("Unsupported condition key '%s', please validate your policy document.", conditionType)
 				return err
 			}
+			conditionArray, ok := conditionKeyVal[key]
+			if ok && findString(conditions[conditionType][key], conditionArray) {
+				err = fmt.Errorf("Ambigious condition values for key '%s', please validate your policy document.", key)
+				return err
+			}
+			conditionKeyVal[key] = append(conditionKeyVal[key], conditions[conditionType][key])
 		}
 	}
 	return nil
 }
 
-// Validate - validate if request body is of proper JSON and in
-// accordance with policy standards.
-func Validate(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
+// List of actions for which prefixes are not allowed.
+var invalidPrefixActions = map[string]struct{}{
+	"s3:GetBucketLocation":          {},
+	"s3:ListBucket":                 {},
+	"s3:ListBucketMultipartUploads": {},
+	// Add actions which do not honor prefixes.
+}
+
+// resourcePrefix - provides the prefix removing any wildcards.
+func resourcePrefix(resource string) string {
+	if strings.HasSuffix(resource, "*") {
+		resource = strings.TrimSuffix(resource, "*")
+	}
+	return path.Clean(resource)
+}
+
+// checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
+// First valation of Resources done for given set of Actions.
+// Later its validated for recursive Resources.
+func checkBucketPolicyResources(bucket string, bucketPolicy BucketPolicy) APIErrorCode {
+	// Validate statements for special actions and collect resources
+	// for others to validate nesting.
+	var resourceMap = make(map[string]struct{})
+	for _, statement := range bucketPolicy.Statements {
+		for _, action := range statement.Actions {
+			for _, resource := range statement.Resources {
+				resourcePrefix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+				if _, ok := invalidPrefixActions[action]; ok {
+					// Resource prefix is not equal to bucket for
+					// prefix invalid actions, reject them.
+					if resourcePrefix != bucket {
+						return ErrMalformedPolicy
+					}
+				} else {
+					// For all other actions validate if resourcePrefix begins
+					// with bucket name, if not reject them.
+					if strings.Split(resourcePrefix, "/")[0] != bucket {
+						return ErrMalformedPolicy
+					}
+					// All valid resources collect them separately to verify nesting.
+					resourceMap[resourcePrefix] = struct{}{}
+				}
+			}
+		}
+	}
+
+	var resources []string
+	for resource := range resourceMap {
+		resources = append(resources, resourcePrefix(resource))
+	}
+
+	// Sort strings as shorter first.
+	sort.Strings(resources)
+
+	for len(resources) > 1 {
+		var resource string
+		resource, resources = resources[0], resources[1:]
+		// Loop through all resources, if one of them matches with
+		// previous shorter one, it means we have detected
+		// nesting. Reject such rules.
+		for _, otherResource := range resources {
+			// Common prefix reject such rules.
+			if strings.HasPrefix(otherResource, resource) {
+				return ErrMalformedPolicy
+			}
+		}
+	}
+
+	// No errors found.
+	return ErrNone
+}
+
+// parseBucketPolicy - parses and validates if bucket policy is of
+// proper JSON and follows allowed restrictions with policy standards.
+func parseBucketPolicy(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
 	if err = json.Unmarshal(bucketPolicyBuf, &policy); err != nil {
 		return BucketPolicy{}, err
 	}
@@ -216,7 +297,7 @@ func Validate(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
 		if err := isValidActions(statement.Actions); err != nil {
 			return BucketPolicy{}, err
 		}
-		// Statment resources should be valid.
+		// Statement resources should be valid.
 		if err := isValidResources(statement.Resources); err != nil {
 			return BucketPolicy{}, err
 		}
@@ -225,6 +306,23 @@ func Validate(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
 			return BucketPolicy{}, err
 		}
 	}
+
+	// Separate deny and allow statements, so that we can apply deny
+	// statements in the beginning followed by Allow statements.
+	var denyStatements []policyStatement
+	var allowStatements []policyStatement
+	for _, statement := range policy.Statements {
+		if statement.Effect == "Deny" {
+			denyStatements = append(denyStatements, statement)
+			continue
+		}
+		// else if statement.Effect == "Allow"
+		allowStatements = append(allowStatements, statement)
+	}
+
+	// Deny statements are enforced first once matched.
+	policy.Statements = append(denyStatements, allowStatements...)
+
 	// Return successfully parsed policy structure.
 	return policy, nil
 }

bucket-policy-parser_test.go

@@ -0,0 +1,661 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+var (
+	readWriteBucketActions = []string{
+		"s3:GetBucketLocation",
+		"s3:ListBucket",
+		"s3:ListBucketMultipartUploads",
+		// Add more bucket level read-write actions here.
+	}
+	readWriteObjectActions = []string{
+		"s3:AbortMultipartUpload",
+		"s3:DeleteObject",
+		"s3:GetObject",
+		"s3:ListMultipartUploadParts",
+		"s3:PutObject",
+		// Add more object level read-write actions here.
+	}
+)
+
+// Write only actions.
+var (
+	writeOnlyBucketActions = []string{
+		"s3:GetBucketLocation",
+		"s3:ListBucketMultipartUploads",
+		// Add more bucket level write actions here.
+	}
+	writeOnlyObjectActions = []string{
+		"s3:AbortMultipartUpload",
+		"s3:DeleteObject",
+		"s3:ListMultipartUploadParts",
+		"s3:PutObject",
+		// Add more object level write actions here.
+	}
+)
+
+// Read only actions.
+var (
+	readOnlyBucketActions = []string{
+		"s3:GetBucketLocation",
+		"s3:ListBucket",
+		// Add more bucket level read actions here.
+	}
+	readOnlyObjectActions = []string{
+		"s3:GetObject",
+		// Add more object level read actions here.
+	}
+)
+
+// Obtain statements for read-write BucketPolicy.
+func setReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
+	bucketResourceStatement := policyStatement{}
+	objectResourceStatement := policyStatement{}
+	statements := []policyStatement{}
+
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal.AWS = []string{"*"}
+	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
+	bucketResourceStatement.Actions = readWriteBucketActions
+	objectResourceStatement.Effect = "Allow"
+	objectResourceStatement.Principal.AWS = []string{"*"}
+	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
+	objectResourceStatement.Actions = readWriteObjectActions
+	// Save the read write policy.
+	statements = append(statements, bucketResourceStatement, objectResourceStatement)
+	return statements
+}
+
+// Obtain statements for read only BucketPolicy.
+func setReadOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	bucketResourceStatement := policyStatement{}
+	objectResourceStatement := policyStatement{}
+	statements := []policyStatement{}
+
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal.AWS = []string{"*"}
+	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
+	bucketResourceStatement.Actions = readOnlyBucketActions
+	objectResourceStatement.Effect = "Allow"
+	objectResourceStatement.Principal.AWS = []string{"*"}
+	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
+	objectResourceStatement.Actions = readOnlyObjectActions
+	// Save the read only policy.
+	statements = append(statements, bucketResourceStatement, objectResourceStatement)
+	return statements
+}
+
+// Obtain statements for write only BucketPolicy.
+func setWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	bucketResourceStatement := policyStatement{}
+	objectResourceStatement := policyStatement{}
+	statements := []policyStatement{}
+	// Write only policy.
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal.AWS = []string{"*"}
+	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
+	bucketResourceStatement.Actions = writeOnlyBucketActions
+	objectResourceStatement.Effect = "Allow"
+	objectResourceStatement.Principal.AWS = []string{"*"}
+	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
+	objectResourceStatement.Actions = writeOnlyObjectActions
+	// Save the write only policy.
+	statements = append(statements, bucketResourceStatement, objectResourceStatement)
+	return statements
+}
+
+// Tests validate Action validator.
+func TestIsValidActions(t *testing.T) {
+	testCases := []struct {
+		// input.
+		actions []string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Action.
+		// Test case - 1.
+		// "s3:ListObject" is an invalid Action.
+		{[]string{"s3:GetObject", "s3:ListObject", "s3:RemoveObject"}, errors.New("Unsupported action found: ‘s3:ListObject’, please validate your policy document."), false},
+		// Test case - 2.
+		// Empty Actions.
+		{[]string{}, errors.New("Action list cannot be empty."), false},
+		// Test case - 3.
+		// "s3:DeleteEverything"" is an invalid Action.
+		{[]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}, errors.New("Unsupported action found: ‘s3:DeleteEverything’, please validate your policy document."), false},
+
+		// Inputs with valid Action.
+		// Test Case - 4.
+		{[]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidActions(testCase.actions)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+
+	}
+}
+
+// Tests validate Effect validator.
+func TestIsValidEffect(t *testing.T) {
+	testCases := []struct {
+		// input.
+		effect string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Effect.
+		// Test case - 1.
+		{"DontAllow", errors.New("Unsupported Effect found: ‘DontAllow’, please validate your policy document."), false},
+		// Test case - 2.
+		{"NeverAllow", errors.New("Unsupported Effect found: ‘NeverAllow’, please validate your policy document."), false},
+		// Test case - 3.
+		{"AllowAlways", errors.New("Unsupported Effect found: ‘AllowAlways’, please validate your policy document."), false},
+
+		// Inputs with valid Effect.
+		// Test Case - 4.
+		{"Allow", nil, true},
+		// Test Case - 5.
+		{"Deny", nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidEffect(testCase.effect)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+	}
+}
+
+// Tests validate Resources validator.
+func TestIsValidResources(t *testing.T) {
+	testCases := []struct {
+		// input.
+		resources []string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Action.
+		// Test case - 1.
+		// Empty Resources.
+		{[]string{}, errors.New("Resource list cannot be empty."), false},
+		// Test case - 2.
+		// A valid resource should have prefix "arn:aws:s3:::".
+		{[]string{"my-resource"}, errors.New("Unsupported resource style found: ‘my-resource’, please validate your policy document."), false},
+		// Test case - 3.
+		// A Valid resource should have bucket name followed by "arn:aws:s3:::".
+		{[]string{"arn:aws:s3:::"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document."), false},
+		// Test Case - 4.
+		// Valid resource shouldn't have slash('/') followed by "arn:aws:s3:::".
+		{[]string{"arn:aws:s3:::/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document."), false},
+
+		// Test cases with valid Resources.
+		{[]string{"arn:aws:s3:::my-bucket"}, nil, true},
+		{[]string{"arn:aws:s3:::my-bucket/Asia/*"}, nil, true},
+		{[]string{"arn:aws:s3:::my-bucket/Asia/India/*"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidResources(testCase.resources)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+	}
+}
+
+// Tests validate principals validator.
+func TestIsValidPrincipals(t *testing.T) {
+	testCases := []struct {
+		// input.
+		principals []string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Principals.
+		// Test case - 1.
+		// Empty Principals list.
+		{[]string{}, errors.New("Principal cannot be empty."), false},
+		// Test case - 2.
+		// "*" is the only valid principal.
+		{[]string{"my-principal"}, errors.New("Unsupported principal style found: ‘my-principal’, please validate your policy document."), false},
+		// Test case - 3.
+		{[]string{"*", "111122233"}, errors.New("Unsupported principal style found: ‘111122233’, please validate your policy document."), false},
+
+		// Test case - 4.
+		// Test case with valid principal value.
+		{[]string{"*"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidPrincipals(testCase.principals)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+	}
+}
+
+// Tests validate policyStatement condition validator.
+func TestIsValidConditions(t *testing.T) {
+	// returns empty conditions map.
+	setEmptyConditions := func() map[string]map[string]string {
+		return make(map[string]map[string]string)
+	}
+
+	// returns map with the "StringEquals" set to empty map.
+	setEmptyStringEquals := func() map[string]map[string]string {
+		emptyMap := make(map[string]string)
+		conditions := make(map[string]map[string]string)
+		conditions["StringEquals"] = emptyMap
+		return conditions
+
+	}
+
+	// returns map with the "StringNotEquals" set to empty map.
+	setEmptyStringNotEquals := func() map[string]map[string]string {
+		emptyMap := make(map[string]string)
+		conditions := make(map[string]map[string]string)
+		conditions["StringNotEquals"] = emptyMap
+		return conditions
+
+	}
+	// Generate conditions.
+	generateConditions := func(key1, key2, value string) map[string]map[string]string {
+		innerMap := make(map[string]string)
+		innerMap[key2] = value
+		conditions := make(map[string]map[string]string)
+		conditions[key1] = innerMap
+		return conditions
+	}
+
+	// generate ambigious conditions.
+	generateAmbigiousConditions := func() map[string]map[string]string {
+		innerMap := make(map[string]string)
+		innerMap["s3:prefix"] = "Asia/"
+		conditions := make(map[string]map[string]string)
+		conditions["StringEquals"] = innerMap
+		conditions["StringNotEquals"] = innerMap
+		return conditions
+	}
+
+	// generate valid and non valid type in the condition map.
+	generateValidInvalidConditions := func() map[string]map[string]string {
+		innerMap := make(map[string]string)
+		innerMap["s3:prefix"] = "Asia/"
+		conditions := make(map[string]map[string]string)
+		conditions["StringEquals"] = innerMap
+		conditions["InvalidType"] = innerMap
+		return conditions
+	}
+
+	// generate valid and invalid keys for valid types in the same condition map.
+	generateValidInvalidConditionKeys := func() map[string]map[string]string {
+		innerMapValid := make(map[string]string)
+		innerMapValid["s3:prefix"] = "Asia/"
+		innerMapInValid := make(map[string]string)
+		innerMapInValid["s3:invalid"] = "Asia/"
+		conditions := make(map[string]map[string]string)
+		conditions["StringEquals"] = innerMapValid
+		conditions["StringEquals"] = innerMapInValid
+		return conditions
+	}
+
+	// List of Conditions used for test cases.
+	testConditions := []map[string]map[string]string{
+		generateConditions("StringValues", "s3:max-keys", "100"),
+		generateConditions("StringEquals", "s3:Object", "100"),
+		generateAmbigiousConditions(),
+		generateValidInvalidConditions(),
+		generateValidInvalidConditionKeys(),
+		setEmptyConditions(),
+		setEmptyStringEquals(),
+		setEmptyStringNotEquals(),
+		generateConditions("StringEquals", "s3:prefix", "Asia/"),
+		generateConditions("StringEquals", "s3:max-keys", "100"),
+		generateConditions("StringNotEquals", "s3:prefix", "Asia/"),
+		generateConditions("StringNotEquals", "s3:max-keys", "100"),
+	}
+
+	testCases := []struct {
+		inputCondition map[string]map[string]string
+		// expected result.
+		expectedErr error
+		// flag indicating whether test should pass.
+		shouldPass bool
+	}{
+		// Malformed conditions.
+		// Test case - 1.
+		// "StringValues" is an invalid type.
+		{testConditions[0], fmt.Errorf("Unsupported condition type 'StringValues', " +
+			"please validate your policy document."), false},
+		// Test case - 2.
+		// "s3:Object" is an invalid key.
+		{testConditions[1], fmt.Errorf("Unsupported condition key " +
+			"'StringEquals', please validate your policy document."), false},
+		// Test case - 3.
+		// Test case with Ambigious conditions set.
+		{testConditions[2], fmt.Errorf("Ambigious condition values for key 's3:prefix', " +
+			"please validate your policy document."), false},
+		// Test case - 4.
+		// Test case with valid and invalid condition types.
+		{testConditions[3], fmt.Errorf("Unsupported condition type 'InvalidType', " +
+			"please validate your policy document."), false},
+		// Test case - 5.
+		// Test case with valid and invalid condition keys.
+		{testConditions[4], fmt.Errorf("Unsupported condition key 'StringEquals', " +
+			"please validate your policy document."), false},
+		// Test cases with valid conditions.
+		// Test case - 6.
+		{testConditions[5], nil, true},
+		// Test case - 7.
+		{testConditions[6], nil, true},
+		// Test case - 8.
+		{testConditions[7], nil, true},
+		// Test case - 9.
+		{testConditions[8], nil, true},
+		// Test case - 10.
+		{testConditions[9], nil, true},
+		// Test case - 11.
+		{testConditions[10], nil, true},
+		// Test case 10.
+		{testConditions[11], nil, true},
+	}
+	for i, testCase := range testCases {
+		actualErr := isValidConditions(testCase.inputCondition)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, actualErr.Error())
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.expectedErr.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if actualErr.Error() != testCase.expectedErr.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.expectedErr.Error(), actualErr.Error())
+			}
+		}
+	}
+}
+
+// Tests validate Policy Action and Resource fields.
+func TestCheckBucketPolicyResources(t *testing.T) {
+	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
+	setValidPrefixActions := func(statements []policyStatement) []policyStatement {
+		statements[0].Actions = []string{"s3:DeleteObject", "s3:PutObject"}
+		return statements
+	}
+	// contructing policy statement with recursive resources.
+	// should result in ErrMalformedPolicy
+	setRecurseResource := func(statements []policyStatement) []policyStatement {
+		statements[0].Resources = []string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}
+		return statements
+	}
+
+	// constructing policy statement with lexically close characters.
+	// should not result in ErrMalformedPolicy
+	setResourceLexical := func(statements []policyStatement) []policyStatement {
+		statements[0].Resources = []string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}
+		return statements
+	}
+
+	// List of BucketPolicy used for tests.
+	bucketAccessPolicies := []BucketPolicy{
+		// BucketPolicy - 1.
+		// Contains valid read only policy statement.
+		{Version: "1.0", Statements: setReadOnlyStatement("minio-bucket", "")},
+		// BucketPolicy - 2.
+		// Contains valid read-write only policy statement.
+		{Version: "1.0", Statements: setReadWriteStatement("minio-bucket", "Asia/")},
+		// BucketPolicy - 3.
+		// Contains valid write only policy statement.
+		{Version: "1.0", Statements: setWriteOnlyStatement("minio-bucket", "Asia/India/")},
+		// BucketPolicy - 4.
+		// Contains invalidPrefixActions.
+		// Since resourcePrefix is not to the bucket-name, it return ErrMalformedPolicy.
+		{Version: "1.0", Statements: setReadOnlyStatement("minio-bucket-fail", "Asia/India/")},
+		// BucketPolicy - 5.
+		// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
+		// but bucket part of the resource is not equal to the bucket name.
+		// this results in return of ErrMalformedPolicy.
+		{Version: "1.0", Statements: setValidPrefixActions(setWriteOnlyStatement("minio-bucket-fail", "Asia/India/"))},
+		// BucketPolicy - 6.
+		// contructing policy statement with recursive resources.
+		// should result in ErrMalformedPolicy
+		{Version: "1.0", Statements: setRecurseResource(setValidPrefixActions(setWriteOnlyStatement("minio-bucket", "")))},
+		// BucketPolciy - 7.
+		// constructing policy statement with non recursive but
+		// lexically close resources.
+		// should result in ErrNone.
+		{Version: "1.0", Statements: setResourceLexical(setValidPrefixActions(setWriteOnlyStatement("minio-bucket", "oo")))},
+	}
+
+	testCases := []struct {
+		inputPolicy BucketPolicy
+		// expected results.
+		apiErrCode APIErrorCode
+		// Flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Test case - 1.
+		{bucketAccessPolicies[0], ErrNone, true},
+		// Test case - 2.
+		{bucketAccessPolicies[1], ErrNone, true},
+		// Test case - 3.
+		{bucketAccessPolicies[2], ErrNone, true},
+		// Test case - 4.
+		// contains invalidPrefixActions (check bucket-policy-parser.go).
+		// Resource prefix will not be equal to the bucket name in this case.
+		{bucketAccessPolicies[3], ErrMalformedPolicy, false},
+		// Test case - 5.
+		// actions contain invalidPrefixActions (check bucket-policy-parser.go).
+		// Resource prefix bucket part is not equal to the bucket name in this case.
+		{bucketAccessPolicies[4], ErrMalformedPolicy, false},
+		// Test case - 6.
+		// contructing policy statement with recursive resources.
+		// should result in ErrMalformedPolicy.
+		{bucketAccessPolicies[5], ErrMalformedPolicy, false},
+		// Test case - 7.
+		// constructing policy statement with lexically close
+		// characters.
+		// should result in ErrNone.
+		{bucketAccessPolicies[6], ErrNone, true},
+	}
+	for i, testCase := range testCases {
+		apiErrCode := checkBucketPolicyResources("minio-bucket", testCase.inputPolicy)
+		if apiErrCode != ErrNone && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
+		}
+		if apiErrCode == ErrNone && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with ErrCode %v, but passed instead", i+1, testCase.apiErrCode)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if apiErrCode != ErrNone && !testCase.shouldPass {
+			if testCase.apiErrCode != apiErrCode {
+				t.Errorf("Test %d: Expected to fail with error code %v, but instead failed with error code %v", i+1, testCase.apiErrCode, apiErrCode)
+			}
+		}
+	}
+}
+
+// Tests validate parsing of BucketAccessPolicy.
+func TestParseBucketPolicy(t *testing.T) {
+	// set Unsupported Actions.
+	setUnsupportedActions := func(statements []policyStatement) []policyStatement {
+		// "s3:DeleteEverything"" is an Unsupported Action.
+		statements[0].Actions = []string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}
+		return statements
+	}
+	// set unsupported Effect.
+	setUnsupportedEffect := func(statements []policyStatement) []policyStatement {
+		// Effect "Don't allow" is Unsupported.
+		statements[0].Effect = "DontAllow"
+		return statements
+	}
+	// set unsupported principals.
+	setUnsupportedPrincipals := func(statements []policyStatement) []policyStatement {
+		// "User1111"" is an Unsupported Principal.
+		statements[0].Principal.AWS = []string{"*", "User1111"}
+		return statements
+	}
+	// set unsupported Resources.
+	setUnsupportedResources := func(statements []policyStatement) []policyStatement {
+		// "s3:DeleteEverything"" is an Unsupported Action.
+		statements[0].Resources = []string{"my-resource"}
+		return statements
+	}
+	// List of BucketPolicy used for test cases.
+	bucketAccesPolicies := []BucketPolicy{
+		// BucketPolicy - 0.
+		// BucketPolicy statement empty.
+		{Version: "1.0"},
+		// BucketPolicy - 1.
+		// BucketPolicy version empty.
+		{Version: "", Statements: []policyStatement{}},
+		// BucketPolicy - 2.
+		// Readonly BucketPolicy.
+		{Version: "1.0", Statements: setReadOnlyStatement("minio-bucket", "")},
+		// BucketPolicy - 3.
+		// Read-Write bucket policy.
+		{Version: "1.0", Statements: setReadWriteStatement("minio-bucket", "Asia/")},
+		// BucketPolicy - 4.
+		// Write only bucket policy.
+		{Version: "1.0", Statements: setWriteOnlyStatement("minio-bucket", "Asia/India/")},
+		// BucketPolicy - 5.
+		// BucketPolicy statement contains unsupported action.
+		{Version: "1.0", Statements: setUnsupportedActions(setReadOnlyStatement("minio-bucket", ""))},
+		// BucketPolicy - 6.
+		// BucketPolicy statement contains unsupported Effect.
+		{Version: "1.0", Statements: setUnsupportedEffect(setReadWriteStatement("minio-bucket", "Asia/"))},
+		// BucketPolicy - 7.
+		// BucketPolicy statement contains unsupported Principal.
+		{Version: "1.0", Statements: setUnsupportedPrincipals(setWriteOnlyStatement("minio-bucket", "Asia/India/"))},
+		// BucketPolicy - 8.
+		// BucketPolicy statement contains unsupported Resource.
+		{Version: "1.0", Statements: setUnsupportedResources(setWriteOnlyStatement("minio-bucket", "Asia/India/"))},
+	}
+
+	testCases := []struct {
+		inputPolicy BucketPolicy
+		// expected results.
+		expectedPolicy BucketPolicy
+		err            error
+		// Flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Test case - 1.
+		// BucketPolicy statement empty.
+		{bucketAccesPolicies[0], BucketPolicy{}, errors.New("Policy statement cannot be empty."), false},
+		// Test case - 2.
+		// BucketPolicy version empty.
+		{bucketAccesPolicies[1], BucketPolicy{}, errors.New("Policy version cannot be empty."), false},
+		// Test case - 3.
+		// Readonly BucketPolicy.
+		{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
+		// Test case - 4.
+		// Read-Write bucket policy.
+		{bucketAccesPolicies[3], bucketAccesPolicies[3], nil, true},
+		// Test case - 5.
+		// Write only bucket policy.
+		{bucketAccesPolicies[4], bucketAccesPolicies[4], nil, true},
+		// Test case - 6.
+		// BucketPolicy statement contains unsupported action.
+		{bucketAccesPolicies[5], bucketAccesPolicies[5], fmt.Errorf("Unsupported action found: ‘s3:DeleteEverything’, please validate your policy document."), false},
+		// Test case - 7.
+		// BucketPolicy statement contains unsupported Effect.
+		{bucketAccesPolicies[6], bucketAccesPolicies[6], fmt.Errorf("Unsupported Effect found: ‘DontAllow’, please validate your policy document."), false},
+		// Test case - 8.
+		// BucketPolicy statement contains unsupported Principal.
+		{bucketAccesPolicies[7], bucketAccesPolicies[7], fmt.Errorf("Unsupported principal style found: ‘User1111’, please validate your policy document."), false},
+		// Test case - 9.
+		// BucketPolicy statement contains unsupported Resource.
+		{bucketAccesPolicies[8], bucketAccesPolicies[8], fmt.Errorf("Unsupported resource style found: ‘my-resource’, please validate your policy document."), false},
+	}
+	for i, testCase := range testCases {
+		inputPolicyBytes, e := json.Marshal(testCase.inputPolicy)
+		if e != nil {
+			t.Fatalf("Test %d: Couldn't Marshal bucket policy", i+1)
+		}
+
+		actualAccessPolicy, err := parseBucketPolicy(inputPolicyBytes)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if !reflect.DeepEqual(testCase.expectedPolicy, actualAccessPolicy) {
+				t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
+			}
+		}
+	}
+}

bucket-policy.go

@@ -20,128 +20,109 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-
-	"github.com/minio/minio/pkg/fs"
-	"github.com/minio/minio/pkg/probe"
 )
 
 // getBucketsConfigPath - get buckets path.
-func getBucketsConfigPath() (string, *probe.Error) {
+func getBucketsConfigPath() (string, error) {
 	configPath, err := getConfigPath()
 	if err != nil {
-		return "", err.Trace()
+		return "", err
 	}
 	return filepath.Join(configPath, "buckets"), nil
 }
 
-// createBucketsConfigPath - create buckets directory.
-func createBucketsConfigPath() *probe.Error {
+// getBucketConfigPath - get bucket config path.
+func getBucketConfigPath(bucket string) (string, error) {
 	bucketsConfigPath, err := getBucketsConfigPath()
 	if err != nil {
-		return err
-	}
-	if e := os.MkdirAll(bucketsConfigPath, 0700); e != nil {
-		return probe.NewError(e)
-	}
-	return nil
-}
-
-// getBucketConfigPath - get bucket path.
-func getBucketConfigPath(bucket string) (string, *probe.Error) {
-	bucketsConfigPath, err := getBucketsConfigPath()
-	if err != nil {
-		return "", err.Trace()
+		return "", err
 	}
 	return filepath.Join(bucketsConfigPath, bucket), nil
 }
 
-// createBucketConfigPath - create bucket directory.
-func createBucketConfigPath(bucket string) *probe.Error {
+// createBucketConfigPath - create bucket config directory.
+func createBucketConfigPath(bucket string) error {
 	bucketConfigPath, err := getBucketConfigPath(bucket)
 	if err != nil {
 		return err
 	}
-	if e := os.MkdirAll(bucketConfigPath, 0700); e != nil {
-		return probe.NewError(e)
-	}
-	return nil
+	return os.MkdirAll(bucketConfigPath, 0700)
 }
 
 // readBucketPolicy - read bucket policy.
-func readBucketPolicy(bucket string) ([]byte, *probe.Error) {
+func readBucketPolicy(bucket string) ([]byte, error) {
 	// Verify bucket is valid.
-	if !fs.IsValidBucketName(bucket) {
-		return nil, probe.NewError(fs.BucketNameInvalid{Bucket: bucket})
+	if !IsValidBucketName(bucket) {
+		return nil, BucketNameInvalid{Bucket: bucket}
 	}
 
 	bucketConfigPath, err := getBucketConfigPath(bucket)
 	if err != nil {
-		return nil, err.Trace()
+		return nil, err
 	}
 
 	// Get policy file.
 	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
-	if _, e := os.Stat(bucketPolicyFile); e != nil {
-		if os.IsNotExist(e) {
-			return nil, probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket})
+	if _, err = os.Stat(bucketPolicyFile); err != nil {
+		if os.IsNotExist(err) {
+			return nil, BucketPolicyNotFound{Bucket: bucket}
 		}
-		return nil, probe.NewError(e)
+		return nil, err
 	}
-
-	accessPolicyBytes, e := ioutil.ReadFile(bucketPolicyFile)
-	if e != nil {
-		return nil, probe.NewError(e)
-	}
-	return accessPolicyBytes, nil
+	return ioutil.ReadFile(bucketPolicyFile)
 }
 
 // removeBucketPolicy - remove bucket policy.
-func removeBucketPolicy(bucket string) *probe.Error {
+func removeBucketPolicy(bucket string) error {
 	// Verify bucket is valid.
-	if !fs.IsValidBucketName(bucket) {
-		return probe.NewError(fs.BucketNameInvalid{Bucket: bucket})
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
 	}
 
 	bucketConfigPath, err := getBucketConfigPath(bucket)
 	if err != nil {
-		return err.Trace(bucket)
+		return err
 	}
 
 	// Get policy file.
 	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
-	if _, e := os.Stat(bucketPolicyFile); e != nil {
-		if os.IsNotExist(e) {
-			return probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket})
+	if _, err = os.Stat(bucketPolicyFile); err != nil {
+		if os.IsNotExist(err) {
+			return BucketPolicyNotFound{Bucket: bucket}
 		}
-		return probe.NewError(e)
+		return err
+	}
+	if err := os.Remove(bucketPolicyFile); err != nil {
+		return err
 	}
 	return nil
 }
 
 // writeBucketPolicy - save bucket policy.
-func writeBucketPolicy(bucket string, accessPolicyBytes []byte) *probe.Error {
+func writeBucketPolicy(bucket string, accessPolicyBytes []byte) error {
 	// Verify if bucket path legal
-	if !fs.IsValidBucketName(bucket) {
-		return probe.NewError(fs.BucketNameInvalid{Bucket: bucket})
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+
+	// Create bucket config path.
+	if err := createBucketConfigPath(bucket); err != nil {
+		return err
 	}
 
 	bucketConfigPath, err := getBucketConfigPath(bucket)
 	if err != nil {
-		return err.Trace()
+		return err
 	}
 
 	// Get policy file.
 	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
-	if _, e := os.Stat(bucketPolicyFile); e != nil {
-		if !os.IsNotExist(e) {
-			return probe.NewError(e)
+	if _, err := os.Stat(bucketPolicyFile); err != nil {
+		if !os.IsNotExist(err) {
+			return err
 		}
 	}
 
 	// Write bucket policy.
-	if e := ioutil.WriteFile(bucketPolicyFile, accessPolicyBytes, 0600); e != nil {
-		return probe.NewError(e)
-	}
-
-	return nil
+	return ioutil.WriteFile(bucketPolicyFile, accessPolicyBytes, 0600)
 }

build-constants.go

@@ -16,13 +16,18 @@
 
 package main
 
+// DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
+// set through ‘buildscripts/gen-ldflags.go’.
 var (
+	// minioGOPATH - GOPATH value at the time of build.
+	minioGOPATH = ""
+
 	// minioVersion - version time.RFC3339.
 	minioVersion = "DEVELOPMENT.GOGET"
 	// minioReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ.
 	minioReleaseTag = "DEVELOPMENT.GOGET"
 	// minioCommitID - latest commit id.
 	minioCommitID = "DEVELOPMENT.GOGET"
-	// minioShortCommitID - first 12 characters from mcCommitID
+	// minioShortCommitID - first 12 characters from minioCommitID.
 	minioShortCommitID = minioCommitID[:12]
 )

buildscripts/build.sh

@@ -23,7 +23,12 @@ _init() {
     fi
 
     # List of supported architectures
-    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm windows/386 windows/amd64 darwin/amd64'
+    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm windows/386 windows/amd64 darwin/amd64 freebsd/amd64'
+
+    ## System binaries
+    CP=`which cp`
+    SHASUM=`which shasum`
+    SED=`which sed`
 }
 
 go_build() {
@@ -33,15 +38,46 @@ go_build() {
     package=$(go list -f '{{.ImportPath}}')
     echo -n "-->"
     printf "%15s:%s\n" "${osarch}" "${package}"
-    GO15VENDOREXPERIMENT=1 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_str/$os-$arch/$(basename $package).$release_tag
+
+    # Release binary name
+    release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
+    # Release binary downloadable name
+    release_real_bin="$release_str/$os-$arch/$(basename $package)"
+    # Release shasum name
+    release_shasum="$release_str/$os-$arch/$(basename $package).shasum"
+
+    # Go build to build the binary.
+    GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
+
+    # Create copy
+    if [ $os == "windows" ]; then
+        $CP -p $release_bin ${release_real_bin}.exe
+    else
+        $CP -p $release_bin $release_real_bin
+    fi
+
+    # Calculate shasum
+    shasum_str=$(${SHASUM} ${release_bin})
+    echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
 }
 
 main() {
     # Build releases.
     echo "Executing $release_str builds for OS: ${SUPPORTED_OSARCH}"
+    echo  "Choose an OS Arch from the below"
     for osarch in ${SUPPORTED_OSARCH}; do
-        go_build ${osarch}
+        echo ${osarch}
     done
+
+    read -p "If you want to build for all, Just press Enter: " chosen_osarch
+    if [ "$chosen_osarch" = "" ]; then
+        for each_osarch in ${SUPPORTED_OSARCH}; do
+            go_build ${each_osarch}
+        done
+    else
+        go_build ${chosen_osarch}
+    fi
+
 }
 
 # Run main.

buildscripts/checkdeps.sh

@@ -21,7 +21,6 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    YASM_VERSION="1.2.0"
     GO_VERSION="1.6"
     OSX_VERSION="10.8"
     UNAME=$(uname -sm)
@@ -141,6 +140,9 @@ is_supported_os() {
         "Linux")
             os="linux"
             ;;
+        "FreeBSD")
+            os="freebsd"
+            ;;
         "Darwin")
             osx_host_version=$(env sw_vers -productVersion)
             check_version "${osx_host_version}" "${OSX_VERSION}"
@@ -155,7 +157,7 @@ is_supported_os() {
 is_supported_arch() {
     local supported
     case ${UNAME##* } in
-        "x86_64")
+        "x86_64" | "amd64")
             supported=1
             ;;
         "arm"*)
@@ -181,11 +183,6 @@ check_deps() {
     if [ $? -ge 2 ]; then
         MISSING="${MISSING} git"
     fi
-
-    check_version "$(env yasm --version 2>/dev/null | sed 's/^.* \([0-9.]*\).*$/\1/' | head -1)" "${YASM_VERSION}"
-    if [ $? -ge 2 ]; then
-        MISSING="${MISSING} yasm(${YASM_VERSION})"
-    fi
 }
 
 main() {

buildscripts/checkgopath.sh

@@ -19,8 +19,8 @@ _init() {
 
     shopt -s extglob
 
-    PWD=$(pwd)
-    GOPATH=$(go env GOPATH)
+    PWD=$(pwd -P)
+    GOPATH=$(cd "$(go env GOPATH)" ; env pwd -P)
 }
 
 main() {

buildscripts/gen-ldflags.go

@@ -28,10 +28,11 @@ import (
 
 func genLDFlags(version string) string {
 	var ldflagsStr string
-	ldflagsStr = "-X main.minioVersion=" + version + " "
-	ldflagsStr = ldflagsStr + "-X main.minioReleaseTag=" + releaseTag(version) + " "
-	ldflagsStr = ldflagsStr + "-X main.minioCommitID=" + commitID() + " "
-	ldflagsStr = ldflagsStr + "-X main.minioShortCommitID=" + commitID()[:12]
+	ldflagsStr = "-X main.minioVersion=" + version
+	ldflagsStr += " -X main.minioReleaseTag=" + releaseTag(version)
+	ldflagsStr += " -X main.minioCommitID=" + commitID()
+	ldflagsStr += " -X main.minioShortCommitID=" + commitID()[:12]
+	ldflagsStr += " -X main.minioGOPATH=" + os.Getenv("GOPATH")
 	return ldflagsStr
 }
 

certs.go

@@ -0,0 +1,90 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/go-homedir"
+)
+
+// createCertsPath create certs path.
+func createCertsPath() error {
+	certsPath, err := getCertsPath()
+	if err != nil {
+		return err
+	}
+	return os.MkdirAll(certsPath, 0700)
+}
+
+// getCertsPath get certs path.
+func getCertsPath() (string, error) {
+	homeDir, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+	certsPath := filepath.Join(homeDir, globalMinioCertsDir)
+	return certsPath, nil
+}
+
+// mustGetCertsPath must get certs path.
+func mustGetCertsPath() string {
+	certsPath, err := getCertsPath()
+	fatalIf(err, "Failed to get certificate path.")
+	return certsPath
+}
+
+// mustGetCertFile must get cert file.
+func mustGetCertFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioCertFile)
+}
+
+// mustGetKeyFile must get key file.
+func mustGetKeyFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioKeyFile)
+}
+
+// isCertFileExists verifies if cert file exists, returns true if
+// found, false otherwise.
+func isCertFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioCertFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isKeyFileExists verifies if key file exists, returns true if found,
+// false otherwise.
+func isKeyFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioKeyFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isSSL - returns true with both cert and key exists.
+func isSSL() bool {
+	if isCertFileExists() && isKeyFileExists() {
+		return true
+	}
+	return false
+}

commands.go

@@ -18,13 +18,26 @@ package main
 
 import "github.com/minio/cli"
 
-// Collection of minio commands currently supported are
+// Collection of minio commands currently supported are.
 var commands = []cli.Command{}
 
-// Collection of minio commands currently supported in a trie tree
+// Collection of minio commands currently supported in a trie tree.
 var commandsTree = newTrie()
 
-// registerCommand registers a cli command
+// Collection of minio flags currently supported.
+var globalFlags = []cli.Flag{
+	cli.StringFlag{
+		Name:  "config-dir, C",
+		Value: mustGetConfigPath(),
+		Usage: "Path to configuration folder.",
+	},
+	cli.BoolFlag{
+		Name:  "quiet",
+		Usage: "Suppress chatty output.",
+	},
+}
+
+// registerCommand registers a cli command.
 func registerCommand(command cli.Command) {
 	commands = append(commands, command)
 	commandsTree.Insert(command.Name)

config-logger-main.go

@@ -1,164 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"runtime"
-
-	"github.com/minio/cli"
-	"github.com/minio/mc/pkg/console"
-	"github.com/minio/minio/pkg/probe"
-)
-
-// Configure logger
-var configLoggerCmd = cli.Command{
-	Name:   "logger",
-	Usage:  "Configure logger.",
-	Action: mainConfigLogger,
-	CustomHelpTemplate: `NAME:
-   minio config {{.Name}} - {{.Usage}}
-
-USAGE:
-   minio config {{.Name}} OPERATION [ARGS...]
-
-   OPERATION = add | list | remove
-
-EXAMPLES:
-   1. Configure new mongo logger.
-      $ minio config {{.Name}} add mongo localhost:28710 mydb mylogger
-
-   2. Configure new syslog logger. NOTE: syslog logger is not supported on windows.
-      $ minio config {{.Name}} add syslog localhost:554 udp
-
-   3. Configure new file logger. "/var/log" should be writable by user.
-      $ minio config {{.Name}} add file /var/log/minio.log
-
-   4. List currently configured logger.
-      $ minio config {{.Name}} list
-
-   5. Remove/Reset a configured logger.
-      $ minio config {{.Name}} remove mongo
-`,
-}
-
-// Inherit at one place
-type config struct {
-	*configV2
-}
-
-func mainConfigLogger(ctx *cli.Context) {
-	if !ctx.Args().Present() || ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, "logger", 1) // last argument is exit code
-	}
-	conf, err := loadConfigV2()
-	fatalIf(err.Trace(), "Unable to load config", nil)
-
-	if ctx.Args().Get(0) == "add" {
-		args := ctx.Args().Tail()
-		if args.Get(0) == "mongo" {
-			enableLog2Mongo(&config{conf}, args.Tail())
-		}
-		if args.Get(0) == "syslog" {
-			if runtime.GOOS == "windows" {
-				fatalIf(probe.NewError(errInvalidArgument), "Syslog is not supported on windows.", nil)
-			}
-			enableLog2Syslog(&config{conf}, args.Tail())
-		}
-		if args.Get(0) == "file" {
-			enableLog2File(&config{conf}, args.Tail())
-		}
-	}
-	if ctx.Args().Get(0) == "remove" {
-		args := ctx.Args().Tail()
-		if args.Get(0) == "mongo" {
-			conf.MongoLogger.Addr = ""
-			conf.MongoLogger.DB = ""
-			conf.MongoLogger.Collection = ""
-			err := saveConfig(conf)
-			fatalIf(err.Trace(), "Unable to save config.", nil)
-		}
-		if args.Get(0) == "syslog" {
-			if runtime.GOOS == "windows" {
-				fatalIf(probe.NewError(errInvalidArgument), "Syslog is not supported on windows.", nil)
-			}
-			conf.SyslogLogger.Network = ""
-			conf.SyslogLogger.Addr = ""
-			err := saveConfig(conf)
-			fatalIf(err.Trace(), "Unable to save config.", nil)
-		}
-		if args.Get(0) == "file" {
-			conf.FileLogger.Filename = ""
-			err := saveConfig(conf)
-			fatalIf(err.Trace(), "Unable to save config.", nil)
-		}
-	}
-	if ctx.Args().Get(0) == "list" {
-		console.Println(conf)
-	}
-}
-
-func enableLog2Mongo(conf *config, args cli.Args) {
-	if conf.IsFileLoggingEnabled() {
-		console.Infoln("File logging already enabled. Removing automatically by enabling mongo.")
-		conf.FileLogger.Filename = ""
-	}
-	if conf.IsSysloggingEnabled() {
-		console.Infoln("Syslog logging already enabled. Removing automatically by enabling mongo.")
-		conf.SyslogLogger.Addr = ""
-		conf.SyslogLogger.Network = ""
-	}
-	conf.MongoLogger.Addr = args.Get(0)
-	conf.MongoLogger.DB = args.Get(1)
-	conf.MongoLogger.Collection = args.Get(2)
-
-	err := saveConfig(conf.configV2)
-	fatalIf(err.Trace(), "Unable to save mongo logging config.", nil)
-}
-
-func enableLog2Syslog(conf *config, args cli.Args) {
-	if conf.IsFileLoggingEnabled() {
-		console.Infoln("File logging already enabled. Removing automatically by enabling syslog.")
-		conf.FileLogger.Filename = ""
-	}
-	if conf.IsMongoLoggingEnabled() {
-		console.Infoln("Mongo logging already enabled. Removing automatically by enabling syslog.")
-		conf.MongoLogger.Addr = ""
-		conf.MongoLogger.DB = ""
-		conf.MongoLogger.Collection = ""
-	}
-	conf.SyslogLogger.Addr = args.Get(0)
-	conf.SyslogLogger.Network = args.Get(1)
-	err := saveConfig(conf.configV2)
-	fatalIf(err.Trace(), "Unable to save syslog config.", nil)
-}
-
-func enableLog2File(conf *config, args cli.Args) {
-	if conf.IsSysloggingEnabled() {
-		console.Infoln("Syslog logging already enabled. Removing automatically by enabling file logging.")
-		conf.SyslogLogger.Addr = ""
-		conf.SyslogLogger.Network = ""
-	}
-	if conf.IsMongoLoggingEnabled() {
-		console.Infoln("Mongo logging already enabled. Removing automatically by enabling file logging.")
-		conf.MongoLogger.Addr = ""
-		conf.MongoLogger.DB = ""
-		conf.MongoLogger.Collection = ""
-	}
-	conf.FileLogger.Filename = args.Get(0)
-	err := saveConfig(conf.configV2)
-	fatalIf(err.Trace(), "Unable to save file logging config.", nil)
-}

config-main.go

@@ -1,53 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "github.com/minio/cli"
-
-//   Configure minio server
-//
-//   ----
-//   NOTE: that the configure command only writes values to the config file.
-//   It does not use any configuration values from the environment variables.
-//   ----
-//
-var configCmd = cli.Command{
-	Name:   "config",
-	Usage:  "Collection of config management commands.",
-	Action: mainConfig,
-	Subcommands: []cli.Command{
-		configLoggerCmd,
-		configVersionCmd,
-	},
-	CustomHelpTemplate: `NAME:
-  {{.Name}} - {{.Usage}}
-
-USAGE:
-  {{.Name}} {{if .Flags}}[global flags] {{end}}command{{if .Flags}} [command flags]{{end}} [arguments...]
-
-COMMANDS:
-  {{range .Commands}}{{ .Name }}{{ "\t" }}{{.Usage}}
-  {{end}}
-`,
-}
-
-// mainConfig is the handle for "minio config" command. provides sub-commands which write configuration data in json format to config file.
-func mainConfig(ctx *cli.Context) {
-	if !ctx.Args().Present() || ctx.Args().First() == "help" {
-		cli.ShowAppHelp(ctx)
-	}
-}

config-migrate.go

@@ -0,0 +1,148 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/quick"
+)
+
+func migrateConfig() {
+	// Purge all configs with version '1'.
+	purgeV1()
+	// Migrate version '2' to '3'.
+	migrateV2ToV3()
+	// Migrate version '3' to '4'.
+	migrateV3ToV4()
+}
+
+// Version '1' is not supported anymore and deprecated, safe to delete.
+func purgeV1() {
+	cv1, err := loadConfigV1()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘1’.")
+
+	if cv1.Version == "1" {
+		console.Println("Removed unsupported config version ‘1’.")
+		/// Purge old fsUsers.json file
+		configPath, err := getConfigPath()
+		fatalIf(err, "Unable to retrieve config path.")
+
+		configFile := filepath.Join(configPath, "fsUsers.json")
+		os.RemoveAll(configFile)
+	}
+	fatalIf(errors.New(""), "Failed to migrate unrecognized config version ‘"+cv1.Version+"’.")
+}
+
+// Version '2' to '3' config migration adds new fields and re-orders
+// previous fields. Simplifies config for future additions.
+func migrateV2ToV3() {
+	cv2, err := loadConfigV2()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘2’.")
+	if cv2.Version != "2" {
+		return
+	}
+	srvConfig := &configV3{}
+	srvConfig.Version = "3"
+	srvConfig.Addr = ":9000"
+	srvConfig.Credential = credential{
+		AccessKeyID:     cv2.Credentials.AccessKeyID,
+		SecretAccessKey: cv2.Credentials.SecretAccessKey,
+	}
+	srvConfig.Region = cv2.Credentials.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature V4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = consoleLogger{
+		Enable: true,
+		Level:  "fatal",
+	}
+	flogger := fileLogger{}
+	flogger.Level = "error"
+	if cv2.FileLogger.Filename != "" {
+		flogger.Enable = true
+		flogger.Filename = cv2.FileLogger.Filename
+	}
+	srvConfig.Logger.File = flogger
+
+	slogger := syslogLogger{}
+	slogger.Level = "debug"
+	if cv2.SyslogLogger.Addr != "" {
+		slogger.Enable = true
+		slogger.Addr = cv2.SyslogLogger.Addr
+	}
+	srvConfig.Logger.Syslog = slogger
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err, "Unable to initialize config.")
+
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	// Migrate the config.
+	err = qc.Save(configFile)
+	fatalIf(err, "Failed to migrate config from ‘"+cv2.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+
+	console.Println("Migration from version ‘" + cv2.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}
+
+// Version '3' to '4' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV3ToV4() {
+	cv3, err := loadConfigV3()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘3’.")
+	if cv3.Version != "3" {
+		return
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &serverConfigV4{}
+	srvConfig.Version = globalMinioConfigVersion
+	srvConfig.Credential = cv3.Credential
+	srvConfig.Region = cv3.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv3.Logger.Console
+	srvConfig.Logger.File = cv3.Logger.File
+	srvConfig.Logger.Syslog = cv3.Logger.Syslog
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err, "Unable to initialize the quick config.")
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	err = qc.Save(configFile)
+	fatalIf(err, "Failed to migrate config from ‘"+cv3.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+
+	console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}

config-old.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/minio/pkg/quick"
+)
+
+/////////////////// Config V1 ///////////////////
+type configV1 struct {
+	Version         string `json:"version"`
+	AccessKeyID     string `json:"accessKeyId"`
+	SecretAccessKey string `json:"secretAccessKey"`
+}
+
+// loadConfigV1 load config
+func loadConfigV1() (*configV1, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return nil, err
+	}
+	configFile := filepath.Join(configPath, "fsUsers.json")
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV1{}
+	c.Version = "1"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+/////////////////// Config V2 ///////////////////
+type configV2 struct {
+	Version     string `json:"version"`
+	Credentials struct {
+		AccessKeyID     string `json:"accessKeyId"`
+		SecretAccessKey string `json:"secretAccessKey"`
+		Region          string `json:"region"`
+	} `json:"credentials"`
+	MongoLogger struct {
+		Addr       string `json:"addr"`
+		DB         string `json:"db"`
+		Collection string `json:"collection"`
+	} `json:"mongoLogger"`
+	SyslogLogger struct {
+		Network string `json:"network"`
+		Addr    string `json:"addr"`
+	} `json:"syslogLogger"`
+	FileLogger struct {
+		Filename string `json:"filename"`
+	} `json:"fileLogger"`
+}
+
+// loadConfigV2 load config version '2'.
+func loadConfigV2() (*configV2, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV2{}
+	c.Version = "2"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+/////////////////// Config V3 ///////////////////
+
+// backendV3 type.
+type backendV3 struct {
+	Type  string   `json:"type"`
+	Disk  string   `json:"disk,omitempty"`
+	Disks []string `json:"disks,omitempty"`
+}
+
+// loggerV3 type.
+type loggerV3 struct {
+	Console struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+	}
+	File struct {
+		Enable   bool   `json:"enable"`
+		Filename string `json:"fileName"`
+		Level    string `json:"level"`
+	}
+	Syslog struct {
+		Enable bool   `json:"enable"`
+		Addr   string `json:"address"`
+		Level  string `json:"level"`
+	} `json:"syslog"`
+	// Add new loggers here.
+}
+
+// configV3 server configuration version '3'.
+type configV3 struct {
+	Version string `json:"version"`
+
+	// Backend configuration.
+	Backend backendV3 `json:"backend"`
+
+	// http Server configuration.
+	Addr string `json:"address"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV3 `json:"logger"`
+}
+
+// loadConfigV3 load config version '3'.
+func loadConfigV3() (*configV3, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV3{}
+	c.Version = "3"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

config-v4.go

@@ -0,0 +1,197 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"sync"
+
+	"github.com/minio/minio/pkg/quick"
+)
+
+// serverConfigV4 server configuration version '4'.
+type serverConfigV4 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger logger `json:"logger"`
+
+	// Read Write mutex.
+	rwMutex *sync.RWMutex
+}
+
+// initConfig - initialize server config. config version (called only once).
+func initConfig() error {
+	if !isConfigFileExists() {
+		srvCfg := &serverConfigV4{}
+		srvCfg.Version = globalMinioConfigVersion
+		srvCfg.Region = "us-east-1"
+		srvCfg.Credential = mustGenAccessKeys()
+		// Enable console logger by default on a fresh run.
+		srvCfg.Logger.Console = consoleLogger{
+			Enable: true,
+			Level:  "fatal",
+		}
+		srvCfg.rwMutex = &sync.RWMutex{}
+		// Create config path.
+		err := createConfigPath()
+		if err != nil {
+			return err
+		}
+
+		// Create certs path.
+		err = createCertsPath()
+		if err != nil {
+			return err
+		}
+
+		// Save the new config globally.
+		serverConfig = srvCfg
+
+		// Save config into file.
+		return serverConfig.Save()
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return err
+	}
+	srvCfg := &serverConfigV4{}
+	srvCfg.Version = globalMinioConfigVersion
+	srvCfg.rwMutex = &sync.RWMutex{}
+	qc, err := quick.New(srvCfg)
+	if err != nil {
+		return err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return err
+	}
+	// Save the loaded config globally.
+	serverConfig = srvCfg
+	// Set the version properly after the unmarshalled json is loaded.
+	serverConfig.Version = globalMinioConfigVersion
+	return nil
+}
+
+// serverConfig server config.
+var serverConfig *serverConfigV4
+
+// GetVersion get current config version.
+func (s serverConfigV4) GetVersion() string {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Version
+}
+
+/// Logger related.
+
+// SetFileLogger set new file logger.
+func (s *serverConfigV4) SetFileLogger(flogger fileLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.File = flogger
+}
+
+// GetFileLogger get current file logger.
+func (s serverConfigV4) GetFileLogger() fileLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.File
+}
+
+// SetConsoleLogger set new console logger.
+func (s *serverConfigV4) SetConsoleLogger(clogger consoleLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.Console = clogger
+}
+
+// GetConsoleLogger get current console logger.
+func (s serverConfigV4) GetConsoleLogger() consoleLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.Console
+}
+
+// SetSyslogLogger set new syslog logger.
+func (s *serverConfigV4) SetSyslogLogger(slogger syslogLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.Syslog = slogger
+}
+
+// GetSyslogLogger get current syslog logger.
+func (s *serverConfigV4) GetSyslogLogger() syslogLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.Syslog
+}
+
+// SetRegion set new region.
+func (s *serverConfigV4) SetRegion(region string) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Region = region
+}
+
+// GetRegion get current region.
+func (s serverConfigV4) GetRegion() string {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Region
+}
+
+// SetCredentials set new credentials.
+func (s *serverConfigV4) SetCredential(creds credential) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Credential = creds
+}
+
+// GetCredentials get current credentials.
+func (s serverConfigV4) GetCredential() credential {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Credential
+}
+
+// Save config.
+func (s serverConfigV4) Save() error {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+
+	// get config file.
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err
+	}
+
+	// initialize quick.
+	qc, err := quick.New(&s)
+	if err != nil {
+		return err
+	}
+
+	// Save config file.
+	return qc.Save(configFile)
+}

config-version-main.go

@@ -1,50 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"github.com/minio/cli"
-	"github.com/minio/mc/pkg/console"
-)
-
-// Print config version.
-var configVersionCmd = cli.Command{
-	Name:   "version",
-	Usage:  "Print config version.",
-	Action: mainConfigVersion,
-	CustomHelpTemplate: `NAME:
-   minio config {{.Name}} - {{.Usage}}
-
-USAGE:
-   minio config {{.Name}}
-
-`,
-}
-
-func mainConfigVersion(ctx *cli.Context) {
-	if ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, "version", 1) // last argument is exit code
-	}
-
-	config, err := loadConfigV2()
-	fatalIf(err.Trace(), "Unable to load config", nil)
-
-	// convert interface{} back to its original struct
-	newConf := config
-	type Version string
-	console.Println(newConf.Version)
-}

config.go

@@ -0,0 +1,88 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/go-homedir"
+)
+
+// configPath for custom config path only for testing purposes
+var customConfigPath string
+
+// Sets a new config path.
+func setGlobalConfigPath(configPath string) {
+	customConfigPath = configPath
+}
+
+// getConfigPath get server config path
+func getConfigPath() (string, error) {
+	if customConfigPath != "" {
+		return customConfigPath, nil
+	}
+	homeDir, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+	configPath := filepath.Join(homeDir, globalMinioConfigDir)
+	return configPath, nil
+}
+
+// mustGetConfigPath must get server config path.
+func mustGetConfigPath() string {
+	configPath, err := getConfigPath()
+	fatalIf(err, "Unable to get config path.")
+	return configPath
+}
+
+// createConfigPath create server config path.
+func createConfigPath() error {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return err
+	}
+	return os.MkdirAll(configPath, 0700)
+}
+
+// isConfigFileExists - returns true if config file exists.
+func isConfigFileExists() bool {
+	st, err := os.Stat(mustGetConfigFile())
+	// If file exists and is regular return true.
+	if err == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// mustGetConfigFile must get server config file.
+func mustGetConfigFile() string {
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	return configFile
+}
+
+// getConfigFile get server config file.
+func getConfigFile() (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(configPath, globalMinioConfigFile), nil
+}

docker/Dockerfile

@@ -0,0 +1,10 @@
+# use "make dockerimage" to build
+FROM alpine:3.3
+RUN apk add --no-cache ca-certificates bash
+ADD minio.dockerimage /minio
+ADD export /export
+EXPOSE 9000
+COPY start.sh /start.sh
+ENV ALLOW_CONTAINER_ROOT=1
+ENTRYPOINT ["/start.sh"]
+CMD ["/export"]

docker/start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+## Fix this check when we arrive at XL.
+if [ -z "$1" ]; then
+    echo "Invalid arguments"
+    echo "Usage: <export_dir>"
+    exit 1
+else
+    /bin/mkdir -p "$1" && /minio server "$1"
+fi

docs/TBD.md

@@ -0,0 +1,40 @@
+# Server command line SPEC - Fri Apr  1 19:49:56 PDT 2016
+
+## Single disk
+Regular single server, single disk mode.
+```
+$ minio server <disk>
+```
+
+##  Multi disk
+```
+$ minio controller start - Start minio controller.
+```
+Prints a secret token to be used by all parties.
+
+Start all servers without disk with `MINIO_CONTROLLER` env set, start in cluster mode.
+```
+$ MINIO_CONTROLLER=<host>:<token> minio server
+```
+
+## Minio Controller cli.
+
+Set controller host and token.
+```
+$ export MINIO_CONTROLLER=<host>:<token>
+```
+
+Create a cluster from the pool of nodes.
+```
+$ minioctl new <clustername> <ip1>:/mnt/disk1 .. <ip16>:/mnt/disk1
+```
+
+Start the cluster.
+```
+$ minioctl start <clustername>
+```
+
+Stop the cluster
+```
+$ minioctl stop <clustername>
+```

docs/api-bucket-policy.md

@@ -9,14 +9,10 @@ This package implements parsing and validating bucket access policies based on A
 
 ### Supports following set of operations.
 
-    *
-    s3:*
     s3:GetObject
     s3:ListBucket
     s3:PutObject
-    s3:CreateBucket
     s3:GetBucketLocation
-    s3:DeleteBucket
     s3:DeleteObject
     s3:AbortMultipartUpload
     s3:ListBucketMultipartUploads
@@ -31,3 +27,7 @@ Supported applicable condition keys for each conditions.
 
     s3:prefix
     s3:max-keys
+
+### Nested policy support.
+
+Nested policies are not allowed.

docs/backend-multipart.md

@@ -0,0 +1,33 @@
+## Multipart
+
+Each incoming part is uploaded in the following format.
+
+Placeholder uploads.json to indicate a leaf.
+
+ ```EXPORT_DIR/.minio/multipart/BUCKET/PATH/TO/OBJECT/uploads.json```
+
+Incomplete file
+
+ ```EXPORT_DIR/.minio/multipart/BUCKET/PATH/TO/OBJECT/UPLOADID/00000.incomplete```
+
+Actual parts
+
+ ```EXPORT_DIR/.minio/multipart/BUCKET/PATH/TO/OBJECT/UPLOADID/PART_NUMBER.MD5SUM_STRING```
+
+## FS Format.
+
+Each of these parts are concatenated back to a single contigous file.
+
+```EXPORT_DIR/BUCKET/PATH/TO/OBJECT```
+
+## XL Format.
+
+Each of these parts are kept as is in the following format.
+
+Special json file indicate the metata multipart information, essentially list of parts etc.
+
+```EXPORT_DIR/BUCKET/PATH/TO/OBJECT/00000.minio.multipart```
+
+All the parts that were uploaded.
+
+```EXPORT_DIR/BUCKET/PATH/TO/OBJECT/PART_NUMBER.minio.multipart```

docs/backend/fs/format.json

@@ -0,0 +1,4 @@
+{
+  "format": "fs",
+  "version": "1"
+}

docs/backend/fs/fs.json

@@ -0,0 +1,15 @@
+{
+    "version": "1",
+    "format": "fs",
+    "minio": {
+        "release": "DEVELOPMENT.GOGET"
+    },
+    "parts": [
+        {
+            "number": 1,
+            "name": "object1",
+            "size": 29,
+            "eTag": "",
+        },
+    ]
+}

docs/backend/fs/uploads.json

@@ -0,0 +1,10 @@
+{
+    "version": "1",
+    "format": "fs",
+    "uploadIds": [
+        {
+            "uploadID": "id",
+            "startTime": "time",
+        }
+    ]
+}

docs/backend/xl/format.json

@@ -0,0 +1,20 @@
+{
+  "xl": {
+    "jbod": [
+      "8aa2b1bc-0e5a-49e0-8221-05228336b040",
+      "3467a69b-0266-478a-9e10-e819447e4545",
+      "d4a4505b-4e4f-4864-befd-4f36adb0bc66",
+      "592b6583-ca26-47af-b991-ba6d097e34e8",
+      "c7ef69f0-dbf5-4c0e-b167-d30a441bad7e",
+      "f0b36ea3-fe96-4f2b-bced-22c7f33e0e0c",
+      "b83abf39-e39d-4e7b-8e16-6f9953455a48",
+      "7d63dfc9-5441-4243-bd36-de8db0691982",
+      "c1bbffc5-81f9-4251-9398-33a959b3ce37",
+      "64408f94-26e0-4277-9593-2d703f4d5a91"
+    ],
+    "disk": "8aa2b1bc-0e5a-49e0-8221-05228336b040",
+    "version": "1"
+  },
+  "format": "xl",
+  "version": "1"
+}

docs/backend/xl/uploads.json

@@ -0,0 +1,10 @@
+{
+    "version": "1",
+    "format": "xl",
+    "uploadIds": [
+        {
+            "uploadID": "id",
+            "startTime": "time",
+        }
+    ]
+}

docs/backend/xl/xl.json

@@ -0,0 +1,57 @@
+{
+    "parts": [
+        {
+            "number": 1,
+            "size": 5242880,
+            "etag": "3565c6e741e69a007a5ac7db893a62b5",
+            "name": "object1"
+        },
+        {
+            "number": 2,
+            "size": 5242880,
+            "etag": "d416712335c280ab1e39498552937764",
+            "name": "object2"
+        },
+        {
+            "number": 3,
+            "size": 4338324,
+            "etag": "8a98c5c54d81c6c95ed9bdcaeb941aaf",
+            "name": "object3"
+        }
+    ],
+    "meta": {
+        "md5Sum": "97586a5290d4f5a41328062d6a7da593-3",
+        "content-type": "application\/octet-stream",
+        "content-encoding": ""
+    },
+    "minio": {
+        "release": "DEVELOPMENT.GOGET"
+    },
+    "erasure": {
+        "algorithm": "klauspost/reedsolomon/vandermonde",
+        "index": 2,
+        "distribution": [ 1, 3, 4, 2, 5, 8, 7, 6, 9 ],
+        "blockSize": 4194304,
+        "parity": 5,
+        "data": 5,
+        "checksum": [
+            {
+                "name": "object1",
+                "algorithm": "sha512",
+                "hash": "d9910e1492446389cfae6fe979db0245f96ca97ca2c7a25cab45805882004479320d866a47ea1f7be6a62625dd4de6caf7816009ef9d62779346d01a221b335c",
+            },
+            {
+                "name": "object2",
+                "algorithm": "sha512",
+                "hash": "d9910e1492446389cfae6fe979db0245f96ca97ca2c7a25cab45805882004479320d866a47ea1f7be6a62625dd4de6caf7816009ef9d62779346d01a221b335c",
+            },
+        ],
+    },
+    "stat": {
+        "version": 0,
+        "modTime": "2016-05-24T00:09:40.122390255Z",
+        "size": 14824084
+    },
+    "format": "xl",
+    "version": "1"
+}

docs/logging.md

@@ -0,0 +1,28 @@
+### Logging.
+
+- `fatalIf` - wrapper function which takes error and prints jsonic error messages.
+- `errorIf` - similar to fatalIf but doesn't exit on err != nil.
+
+Supported logging targets.
+
+- console
+- file
+- syslog
+
+Sample logger section from `~/.minio/config.json`
+```
+		"console": {
+			"enable": true,
+			"level": "error"
+		},
+		"file": {
+			"enable": false,
+			"fileName": "",
+			"level": "error"
+		},
+		"syslog": {
+			"enable": false,
+			"address": "",
+			"level": "error"
+		}
+```

docs/web-browser.md

@@ -19,21 +19,22 @@ minio server <testdir>
 
 JSON RPC namespace is `Web`.
 
-#### Auth Operations
+#### Server operations.
+
+* ServerInfo - fetches current server information, includes memory statistics, minio binary
+  version, golang runtime version and more.
+
+#### Auth operations
 
 * Login - waits for 'username, password' and on success replies a new Json Web Token (JWT).
-* ResetToken - resets token, requires password and token.
-* Logout - currently a dummy operation.
+* SetAuth - change access credentials with new 'username, password'.
+* GetAuth - fetch the current auth from the server.
 
-#### Bucket/Object Operations.
+#### Bucket/Object operations.
 
 * ListBuckets - lists buckets, requires a valid token.
 * ListObjects - lists objects, requires a valid token.
 * MakeBucket - make a new bucket, requires a valid token.
-* GetObjectURL - generates a URL for download access, requires a valid token.
-  (generated URL is valid for 1hr)
-* PutObjectURL - generates a URL for upload access, requies a valid token.
-  (generated URL is valid for 1hr)
-
-#### Server Operations. 
-* DiskInfo - get backend disk statistics.
+* RemoveObject - removes an object from a bucket, requires a valid token.
+* Upload - uploads a new object from the browser, requires a valid token.
+* Download - downloads an object from a bucket, requires a valid token.

docs/xl-file-json.md

@@ -0,0 +1,41 @@
+### file.json
+
+``file.json`` is a special file captured and written by XL storage API layer
+to interpret, manage and extract erasured data from multiple disks.
+
+```json
+{
+    "version": "1.0.0",
+    "stat": {
+        "size": 24256,
+        "modTime": "2016-04-28T00:11:37.843Z",
+        "delete": false,
+        "version": 0
+    },
+    "erasure": {
+        "data": 5,
+        "parity": 5,
+        "blockSize": 4194304
+    ],
+    "minio": {
+        "release": "RELEASE.2016-04-28T00-09-47Z"
+    }
+}
+```
+
+#### JSON meaning.
+
+- "version" // Version of the meta json file.
+
+- "stat" // Stat value of written file.
+
+  - "size"     // Size of the file.
+  - "modTime"  // Modified time of the file.
+  - "deleted"  // Field to track if the file is deleted when disks are down.
+  - "version"  // File version tracked when disks are down.
+
+- "erasure" // Erasure metadata for the written file.
+
+  - "data"       // Data blocks parts of the file.
+  - "parity"     // Parity blocks parts of the file.
+  - "blockSize"  // BlockSize read/write chunk size.

erasure-createfile.go

@@ -0,0 +1,149 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/hex"
+	"hash"
+	"io"
+	"sync"
+
+	"github.com/klauspost/reedsolomon"
+)
+
+// erasureCreateFile - writes an entire stream by erasure coding to
+// all the disks, writes also calculate individual block's checksum
+// for future bit-rot protection.
+func erasureCreateFile(disks []StorageAPI, volume string, path string, partName string, data io.Reader, eInfos []erasureInfo) (newEInfos []erasureInfo, size int64, err error) {
+	// Allocated blockSized buffer for reading.
+	buf := make([]byte, blockSizeV1)
+	hashWriters := newHashWriters(len(disks))
+
+	// Just pick one eInfo.
+	eInfo := pickValidErasureInfo(eInfos)
+
+	// Read until io.EOF, erasure codes data and writes to all disks.
+	for {
+		var n int
+		n, err = io.ReadFull(data, buf)
+		if err == io.EOF {
+			break
+		}
+		if err != nil && err != io.ErrUnexpectedEOF {
+			return nil, 0, err
+		}
+		size += int64(n)
+		var blocks [][]byte
+		// Returns encoded blocks.
+		blocks, err = encodeData(buf[:n], eInfo.DataBlocks, eInfo.ParityBlocks)
+		if err != nil {
+			return nil, 0, err
+		}
+		err = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters)
+		if err != nil {
+			return nil, 0, err
+		}
+	}
+
+	// Save the checksums.
+	checkSums := make([]checkSumInfo, len(disks))
+	for index := range disks {
+		blockIndex := eInfo.Distribution[index] - 1
+		checkSums[blockIndex] = checkSumInfo{
+			Name:      partName,
+			Algorithm: "sha512",
+			Hash:      hex.EncodeToString(hashWriters[blockIndex].Sum(nil)),
+		}
+	}
+
+	// Erasure info update for checksum for each disks.
+	newEInfos = make([]erasureInfo, len(disks))
+	for index, eInfo := range eInfos {
+		if eInfo.IsValid() {
+			blockIndex := eInfo.Distribution[index] - 1
+			newEInfos[index] = eInfo
+			newEInfos[index].Checksum = append(newEInfos[index].Checksum, checkSums[blockIndex])
+		}
+	}
+
+	// Return newEInfos.
+	return newEInfos, size, nil
+}
+
+// encodeData - encodes incoming data buffer into
+// dataBlocks+parityBlocks returns a 2 dimensional byte array.
+func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) {
+	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
+	if err != nil {
+		return nil, err
+	}
+	// Split the input buffer into data and parity blocks.
+	var blocks [][]byte
+	blocks, err = rs.Split(dataBuffer)
+	if err != nil {
+		return nil, err
+	}
+
+	// Encode parity blocks using data blocks.
+	err = rs.Encode(blocks)
+	if err != nil {
+		return nil, err
+	}
+
+	// Return encoded blocks.
+	return blocks, nil
+}
+
+// appendFile - append data buffer at path.
+func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, distribution []int, hashWriters []hash.Hash) (err error) {
+	var wg = &sync.WaitGroup{}
+	var wErrs = make([]error, len(disks))
+	// Write encoded data to quorum disks in parallel.
+	for index, disk := range disks {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		// Write encoded data in routine.
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			// Pick the block from the distribution.
+			blockIndex := distribution[index] - 1
+			n, wErr := disk.AppendFile(volume, path, enBlocks[blockIndex])
+			if wErr != nil {
+				wErrs[index] = wErr
+				return
+			}
+			if n != int64(len(enBlocks[blockIndex])) {
+				wErrs[index] = errUnexpected
+				return
+			}
+
+			// Calculate hash for each blocks.
+			hashWriters[blockIndex].Write(enBlocks[blockIndex])
+
+			// Successfully wrote.
+			wErrs[index] = nil
+		}(index, disk)
+	}
+
+	// Wait for all the appends to finish.
+	wg.Wait()
+
+	// Return success.
+	return nil
+}

erasure-readfile.go

@@ -0,0 +1,208 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/hex"
+	"errors"
+
+	"github.com/klauspost/reedsolomon"
+)
+
+// erasureReadFile - read an entire erasure coded file at into a byte
+// array. Erasure coded parts are often few mega bytes in size and it
+// is convenient to return them as byte slice. This function also
+// supports bit-rot detection by verifying checksum of individual
+// block's checksum.
+func erasureReadFile(disks []StorageAPI, volume string, path string, partName string, size int64, eInfos []erasureInfo) ([]byte, error) {
+	// Return data buffer.
+	var buffer []byte
+
+	// Total size left
+	totalSizeLeft := size
+
+	// Starting offset for reading.
+	startOffset := int64(0)
+
+	// Gather previously calculated block checksums.
+	blockCheckSums := metaPartBlockChecksums(disks, eInfos, partName)
+
+	// Pick one erasure info.
+	eInfo := pickValidErasureInfo(eInfos)
+
+	// Write until each parts are read and exhausted.
+	for totalSizeLeft > 0 {
+		// Calculate the proper block size.
+		var curBlockSize int64
+		if eInfo.BlockSize < totalSizeLeft {
+			curBlockSize = eInfo.BlockSize
+		} else {
+			curBlockSize = totalSizeLeft
+		}
+
+		// Calculate the current encoded block size.
+		curEncBlockSize := getEncodedBlockLen(curBlockSize, eInfo.DataBlocks)
+		offsetEncOffset := getEncodedBlockLen(startOffset, eInfo.DataBlocks)
+
+		// Allocate encoded blocks up to storage disks.
+		enBlocks := make([][]byte, len(disks))
+
+		// Counter to keep success data blocks.
+		var successDataBlocksCount = 0
+		var noReconstruct bool // Set for no reconstruction.
+
+		// Read from all the disks.
+		for index, disk := range disks {
+			blockIndex := eInfo.Distribution[index] - 1
+			if !isValidBlock(disks, volume, path, toDiskIndex(blockIndex, eInfo.Distribution), blockCheckSums) {
+				continue
+			}
+			if disk == nil {
+				continue
+			}
+			// Initialize shard slice and fill the data from each parts.
+			enBlocks[blockIndex] = make([]byte, curEncBlockSize)
+			// Read the necessary blocks.
+			_, err := disk.ReadFile(volume, path, offsetEncOffset, enBlocks[blockIndex])
+			if err != nil {
+				enBlocks[blockIndex] = nil
+			}
+			// Verify if we have successfully read all the data blocks.
+			if blockIndex < eInfo.DataBlocks && enBlocks[blockIndex] != nil {
+				successDataBlocksCount++
+				// Set when we have all the data blocks and no
+				// reconstruction is needed, so that we can avoid
+				// erasure reconstruction.
+				noReconstruct = successDataBlocksCount == eInfo.DataBlocks
+				if noReconstruct {
+					// Break out we have read all the data blocks.
+					break
+				}
+			}
+		}
+
+		// Check blocks if they are all zero in length, we have corruption return error.
+		if checkBlockSize(enBlocks) == 0 {
+			return nil, errXLDataCorrupt
+		}
+
+		// Verify if reconstruction is needed, proceed with reconstruction.
+		if !noReconstruct {
+			err := decodeData(enBlocks, eInfo.DataBlocks, eInfo.ParityBlocks)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// Get data blocks from encoded blocks.
+		dataBlocks, err := getDataBlocks(enBlocks, eInfo.DataBlocks, int(curBlockSize))
+		if err != nil {
+			return nil, err
+		}
+
+		// Copy data blocks.
+		buffer = append(buffer, dataBlocks...)
+
+		// Negate the 'n' size written to client.
+		totalSizeLeft -= int64(len(dataBlocks))
+
+		// Increase the offset to move forward.
+		startOffset += int64(len(dataBlocks))
+
+		// Relenquish memory.
+		dataBlocks = nil
+	}
+	return buffer, nil
+}
+
+// PartObjectChecksum - returns the checksum for the part name from the checksum slice.
+func (e erasureInfo) PartObjectChecksum(partName string) checkSumInfo {
+	for _, checksum := range e.Checksum {
+		if checksum.Name == partName {
+			return checksum
+		}
+	}
+	return checkSumInfo{}
+}
+
+// xlMetaPartBlockChecksums - get block checksums for a given part.
+func metaPartBlockChecksums(disks []StorageAPI, eInfos []erasureInfo, partName string) (blockCheckSums []checkSumInfo) {
+	for index := range disks {
+		if eInfos[index].IsValid() {
+			// Save the read checksums for a given part.
+			blockCheckSums = append(blockCheckSums, eInfos[index].PartObjectChecksum(partName))
+		} else {
+			blockCheckSums = append(blockCheckSums, checkSumInfo{})
+		}
+	}
+	return blockCheckSums
+}
+
+// Takes block index and block distribution to get the disk index.
+func toDiskIndex(blockIdx int, distribution []int) (diskIndex int) {
+	diskIndex = -1
+	// Find out the right disk index for the input block index.
+	for index, blockIndex := range distribution {
+		if blockIndex == blockIdx {
+			diskIndex = index
+		}
+	}
+	return diskIndex
+}
+
+// isValidBlock - calculates the checksum hash for the block and
+// validates if its correct returns true for valid cases, false otherwise.
+func isValidBlock(disks []StorageAPI, volume, path string, diskIndex int, blockCheckSums []checkSumInfo) bool {
+	// Unknown block index requested, treat it as error.
+	if diskIndex == -1 {
+		return false
+	}
+	// Disk is not present, treat entire block to be non existent.
+	if disks[diskIndex] == nil {
+		return false
+	}
+	// Read everything for a given block and calculate hash.
+	hashWriter := newHash(blockCheckSums[diskIndex].Algorithm)
+	hashBytes, err := hashSum(disks[diskIndex], volume, path, hashWriter)
+	if err != nil {
+		return false
+	}
+	return hex.EncodeToString(hashBytes) == blockCheckSums[diskIndex].Hash
+}
+
+// decodeData - decode encoded blocks.
+func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
+	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
+	if err != nil {
+		return err
+	}
+	err = rs.Reconstruct(enBlocks)
+	if err != nil {
+		return err
+	}
+	// Verify reconstructed blocks (parity).
+	ok, err := rs.Verify(enBlocks)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		// Blocks cannot be reconstructed, corrupted data.
+		err = errors.New("Verification failed after reconstruction, data likely corrupted.")
+		return err
+	}
+	return nil
+}

erasure-utils.go

@@ -0,0 +1,108 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/sha512"
+	"hash"
+	"io"
+
+	"github.com/klauspost/reedsolomon"
+)
+
+// newHashWriters - inititialize a slice of hashes for the disk count.
+func newHashWriters(diskCount int) []hash.Hash {
+	hashWriters := make([]hash.Hash, diskCount)
+	for index := range hashWriters {
+		hashWriters[index] = newHash("sha512")
+	}
+	return hashWriters
+}
+
+// newHash - gives you a newly allocated hash depending on the input algorithm.
+func newHash(algo string) hash.Hash {
+	switch algo {
+	case "sha512":
+		return sha512.New()
+	// Add new hashes here.
+	default:
+		return sha512.New()
+	}
+}
+
+func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
+	startOffset := int64(0)
+	// Read until io.EOF.
+	for {
+		buf := make([]byte, blockSizeV1)
+		n, err := disk.ReadFile(volume, path, startOffset, buf)
+		if err == io.EOF {
+			break
+		}
+		if err != nil && err != io.EOF {
+			return nil, err
+		}
+		writer.Write(buf[:n])
+		startOffset += n
+	}
+	return writer.Sum(nil), nil
+}
+
+// getDataBlocks - fetches the data block only part of the input encoded blocks.
+func getDataBlocks(enBlocks [][]byte, dataBlocks int, curBlockSize int) (data []byte, err error) {
+	if len(enBlocks) < dataBlocks {
+		return nil, reedsolomon.ErrTooFewShards
+	}
+	size := 0
+	blocks := enBlocks[:dataBlocks]
+	for _, block := range blocks {
+		size += len(block)
+	}
+	if size < curBlockSize {
+		return nil, reedsolomon.ErrShortData
+	}
+
+	write := curBlockSize
+	for _, block := range blocks {
+		if write < len(block) {
+			data = append(data, block[:write]...)
+			return data, nil
+		}
+		data = append(data, block...)
+		write -= len(block)
+	}
+	return data, nil
+}
+
+// checkBlockSize return the size of a single block.
+// The first non-zero size is returned,
+// or 0 if all blocks are size 0.
+func checkBlockSize(blocks [][]byte) int {
+	for _, block := range blocks {
+		if len(block) != 0 {
+			return len(block)
+		}
+	}
+	return 0
+}
+
+// calculate the blockSize based on input length and total number of
+// data blocks.
+func getEncodedBlockLen(inputLen int64, dataBlocks int) (curEncBlockSize int64) {
+	curEncBlockSize = (inputLen + int64(dataBlocks) - 1) / int64(dataBlocks)
+	return curEncBlockSize
+}

erasure-waitcloser.go

@@ -0,0 +1,74 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io"
+	"sync"
+)
+
+// waitCloser implements a Closer that blocks until released, this
+// prevents the writer closing one end of a pipe while making sure
+// that all data is written and committed to disk on the end.
+// Additionally this also implements Write().
+type waitCloser struct {
+	wg     *sync.WaitGroup // Waitgroup for atomicity.
+	writer io.WriteCloser  // Embedded writer.
+}
+
+// Write to the underlying writer.
+func (b *waitCloser) Write(data []byte) (int, error) {
+	return b.writer.Write(data)
+}
+
+// Close blocks until another goroutine calls Release(error). Returns
+// error code if either writer fails or Release is called with an error.
+func (b *waitCloser) Close() error {
+	err := b.writer.Close()
+	b.wg.Wait()
+	return err
+}
+
+// CloseWithError closes the writer; subsequent read to the read
+// half of the pipe will return the error err.
+func (b *waitCloser) CloseWithError(err error) error {
+	w, ok := b.writer.(*io.PipeWriter)
+	if ok {
+		return w.CloseWithError(err)
+	}
+	return err
+}
+
+// release the Close, causing it to unblock. Only call this
+// once. Calling it multiple times results in a panic.
+func (b *waitCloser) release() {
+	b.wg.Done()
+	return
+}
+
+// newWaitCloser creates a new write closer that must be
+// released by the read consumer.
+func newWaitCloser(writer io.WriteCloser) *waitCloser {
+	// Wait group for the go-routine.
+	wg := &sync.WaitGroup{}
+	// Add to the wait group to wait for.
+	wg.Add(1)
+	return &waitCloser{
+		wg:     wg,
+		writer: writer,
+	}
+}

erasure.go

@@ -1,5 +1,3 @@
-// +build !cgo arm
-
 /*
  * Minio Cloud Storage, (C) 2016 Minio, Inc.
  *
@@ -16,8 +14,4 @@
  * limitations under the License.
  */
 
-package sha1
-
-func block(dig *digest, p []byte) {
-	blockGeneric(dig, p)
-}
+package main

flags.go

@@ -1,57 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "github.com/minio/cli"
-
-// Collection of minio flags currently supported
-var flags = []cli.Flag{}
-
-var (
-	configFolderFlag = cli.StringFlag{
-		Name:  "config-folder, C",
-		Value: mustGetConfigPath(),
-		Usage: "Path to configuration folder.",
-	}
-
-	addressFlag = cli.StringFlag{
-		Name:  "address",
-		Value: ":9000",
-		Usage: "ADDRESS:PORT for cloud storage access.",
-	}
-
-	accessLogFlag = cli.BoolFlag{
-		Name:  "enable-accesslog",
-		Hide:  true,
-		Usage: "Enable access logs for all incoming HTTP request.",
-	}
-
-	certFlag = cli.StringFlag{
-		Name:  "cert",
-		Usage: "Provide your domain certificate.",
-	}
-
-	keyFlag = cli.StringFlag{
-		Name:  "key",
-		Usage: "Provide your domain private key.",
-	}
-)
-
-// registerFlag registers a cli flag
-func registerFlag(flag cli.Flag) {
-	flags = append(flags, flag)
-}

format-config-v1.go

@@ -0,0 +1,588 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/skyrings/skyring-common/tools/uuid"
+)
+
+// fsFormat - structure holding 'fs' format.
+type fsFormat struct {
+	Version string `json:"version"`
+}
+
+// xlFormat - structure holding 'xl' format.
+type xlFormat struct {
+	Version string `json:"version"` // Version of 'xl' format.
+	Disk    string `json:"disk"`    // Disk field carries assigned disk uuid.
+	// JBOD field carries the input disk order generated the first
+	// time when fresh disks were supplied.
+	JBOD []string `json:"jbod"`
+}
+
+// formatConfigV1 - structure holds format config version '1'.
+type formatConfigV1 struct {
+	Version string `json:"version"` // Version of the format config.
+	// Format indicates the backend format type, supports two values 'xl' and 'fs'.
+	Format string    `json:"format"`
+	FS     *fsFormat `json:"fs,omitempty"` // FS field holds fs format.
+	XL     *xlFormat `json:"xl,omitempty"` // XL field holds xl format.
+}
+
+/*
+
+All disks online
+-----------------
+- All Unformatted - format all and return success.
+- Some Unformatted - format all and return success.
+- Any JBOD inconsistent - return failure // Requires deep inspection, phase2.
+- Some are corrupt (missing format.json) - return failure  // Requires deep inspection, phase2.
+- Any unrecognized disks - return failure
+
+Some disks are offline and we have quorum.
+-----------------
+- Some unformatted - no heal, return success.
+- Any JBOD inconsistent - return failure // Requires deep inspection, phase2.
+- Some are corrupt (missing format.json) - return failure  // Requires deep inspection, phase2.
+- Any unrecognized disks - return failure
+
+No read quorum
+-----------------
+failure for all cases.
+
+// Pseudo code for managing `format.json`.
+
+// Generic checks.
+if (no quorum) return error
+if (any disk is corrupt) return error // phase2
+if (jbod inconsistent) return error // phase2
+if (disks not recognized) // Always error.
+
+// Specific checks.
+if (all disks online)
+  if (all disks return format.json)
+     if (jbod consistent)
+        if (all disks recognized)
+          return
+  else
+     if (all disks return format.json not found)
+        (initialize format)
+        return
+     else (some disks return format.json not found)
+        (heal format)
+        return
+     fi
+   fi
+else // No healing at this point forward, some disks are offline or dead.
+   if (some disks return format.json not found)
+      if (with force)
+         // Offline disks are marked as dead.
+         (heal format) // Offline disks should be marked as dead.
+         return success
+      else (without force)
+         // --force is necessary to heal few drives, because some drives
+         // are offline. Offline disks will be marked as dead.
+         return error
+      fi
+fi
+*/
+
+var errSomeDiskUnformatted = errors.New("some disks are found to be unformatted")
+var errSomeDiskOffline = errors.New("some disks are offline")
+
+// Returns error slice into understandable errors.
+func reduceFormatErrs(errs []error, diskCount int) error {
+	var errUnformattedDiskCount = 0
+	var errDiskNotFoundCount = 0
+	for _, err := range errs {
+		if err == errUnformattedDisk {
+			errUnformattedDiskCount++
+		} else if err == errDiskNotFound {
+			errDiskNotFoundCount++
+		}
+	}
+	// Returns errUnformattedDisk if all disks report unFormattedDisk.
+	if errUnformattedDiskCount == diskCount {
+		return errUnformattedDisk
+	} else if errUnformattedDiskCount < diskCount && errDiskNotFoundCount == 0 {
+		// Only some disks return unFormattedDisk and all disks are online.
+		return errSomeDiskUnformatted
+	} else if errUnformattedDiskCount < diskCount && errDiskNotFoundCount > 0 {
+		// Only some disks return unFormattedDisk and some disks are
+		// offline as well.
+		return errSomeDiskOffline
+	}
+	return nil
+}
+
+// loadAllFormats - load all format config from all input disks in parallel.
+func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
+	// Initialize sync waitgroup.
+	var wg = &sync.WaitGroup{}
+
+	// Initialize list of errors.
+	var sErrs = make([]error, len(bootstrapDisks))
+
+	// Initialize format configs.
+	var formatConfigs = make([]*formatConfigV1, len(bootstrapDisks))
+
+	// Make a volume entry on all underlying storage disks.
+	for index, disk := range bootstrapDisks {
+		wg.Add(1)
+		// Make a volume inside a go-routine.
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			formatConfig, lErr := loadFormat(disk)
+			if lErr != nil {
+				sErrs[index] = lErr
+				return
+			}
+			formatConfigs[index] = formatConfig
+		}(index, disk)
+	}
+
+	// Wait for all make vol to finish.
+	wg.Wait()
+
+	for _, err := range sErrs {
+		if err != nil {
+			// Return all formats and errors.
+			return formatConfigs, sErrs
+		}
+	}
+	// Return all formats and nil
+	return formatConfigs, nil
+}
+
+// genericFormatCheck - validates and returns error.
+// if (no quorum) return error
+// if (any disk is corrupt) return error // phase2
+// if (jbod inconsistent) return error // phase2
+// if (disks not recognized) // Always error.
+func genericFormatCheck(formatConfigs []*formatConfigV1, sErrs []error) (err error) {
+	// Calculate the errors.
+	var (
+		errCorruptFormatCount = 0
+		errCount              = 0
+	)
+
+	// Through all errors calculate the actual errors.
+	for _, lErr := range sErrs {
+		if lErr == nil {
+			continue
+		}
+		// These errors are good conditions, means disk is online.
+		if lErr == errUnformattedDisk || lErr == errVolumeNotFound {
+			continue
+		}
+		if lErr == errCorruptedFormat {
+			errCorruptFormatCount++
+		} else {
+			errCount++
+		}
+	}
+
+	// Calculate read quorum.
+	readQuorum := len(formatConfigs)/2 + 1
+
+	// Validate the err count under tolerant limit.
+	if errCount > len(formatConfigs)-readQuorum {
+		return errXLReadQuorum
+	}
+
+	// One of the disk has corrupt format, return error.
+	if errCorruptFormatCount > 0 {
+		return errCorruptedFormat
+	}
+
+	// Validates if format and JBOD are consistent across all disks.
+	if err = checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	// Success..
+	return nil
+}
+
+// checkDisksConsistency - checks if all disks are consistent with all JBOD entries on all disks.
+func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
+	var disks = make([]string, len(formatConfigs))
+	var disksFound = make(map[string]bool)
+	// Collect currently available disk uuids.
+	for index, formatConfig := range formatConfigs {
+		if formatConfig == nil {
+			continue
+		}
+		disks[index] = formatConfig.XL.Disk
+	}
+	// Validate collected uuids and verify JBOD.
+	for index, uuid := range disks {
+		if uuid == "" {
+			continue
+		}
+		var formatConfig = formatConfigs[index]
+		for _, savedUUID := range formatConfig.XL.JBOD {
+			if savedUUID == uuid {
+				disksFound[uuid] = true
+			}
+		}
+	}
+	// Check if all disks are found.
+	for _, value := range disksFound {
+		if !value {
+			return errors.New("Some disks not found in JBOD.")
+		}
+	}
+	return nil
+}
+
+// checkJBODConsistency - validate xl jbod order if they are consistent.
+func checkJBODConsistency(formatConfigs []*formatConfigV1) error {
+	var firstJBOD []string
+	// Extract first valid JBOD.
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		firstJBOD = format.XL.JBOD
+		break
+	}
+	jbodStr := strings.Join(firstJBOD, ".")
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		savedJBODStr := strings.Join(format.XL.JBOD, ".")
+		if jbodStr != savedJBODStr {
+			return errors.New("Inconsistent JBOD found.")
+		}
+	}
+	return nil
+}
+
+func findIndex(disk string, jbod []string) int {
+	for index, uuid := range jbod {
+		if uuid == disk {
+			return index
+		}
+	}
+	return -1
+}
+
+// reorderDisks - reorder disks in JBOD order.
+func reorderDisks(bootstrapDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
+	var savedJBOD []string
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		savedJBOD = format.XL.JBOD
+		break
+	}
+	// Pick the first JBOD list to verify the order and construct new set of disk slice.
+	var newDisks = make([]StorageAPI, len(bootstrapDisks))
+	for fIndex, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		jIndex := findIndex(format.XL.Disk, savedJBOD)
+		if jIndex == -1 {
+			return nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found")
+		}
+		newDisks[jIndex] = bootstrapDisks[fIndex]
+	}
+	return newDisks, nil
+}
+
+// loadFormat - load format from disk.
+func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
+	var buffer []byte
+	buffer, err = readAll(disk, minioMetaBucket, formatConfigFile)
+	if err != nil {
+		// 'file not found' and 'volume not found' as
+		// same. 'volume not found' usually means its a fresh disk.
+		if err == errFileNotFound || err == errVolumeNotFound {
+			var vols []VolInfo
+			vols, err = disk.ListVols()
+			if err != nil {
+				return nil, err
+			}
+			if len(vols) > 1 {
+				// 'format.json' not found, but we found user data.
+				return nil, errCorruptedFormat
+			}
+			// No other data found, its a fresh disk.
+			return nil, errUnformattedDisk
+		}
+		return nil, err
+	}
+	format = &formatConfigV1{}
+	err = json.Unmarshal(buffer, format)
+	if err != nil {
+		return nil, err
+	}
+	return format, nil
+}
+
+// Heals any missing format.json on the drives. Returns error only for unexpected errors
+// as regular errors can be ignored since there might be enough quorum to be operational.
+func healFormatXL(bootstrapDisks []StorageAPI) error {
+	needHeal := make([]bool, len(bootstrapDisks)) // Slice indicating which drives needs healing.
+
+	formatConfigs := make([]*formatConfigV1, len(bootstrapDisks))
+	var referenceConfig *formatConfigV1
+	successCount := 0        // Tracks if we have successfully loaded all `format.json` from all disks.
+	formatNotFoundCount := 0 // Tracks if we `format.json` is not found on all disks.
+	// Loads `format.json` from all disks.
+	for index, disk := range bootstrapDisks {
+		formatXL, err := loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk {
+				// format.json is missing, should be healed.
+				needHeal[index] = true
+				formatNotFoundCount++
+				continue
+			} else if err == errDiskNotFound { // Is a valid case we
+				// can proceed without healing.
+				return nil
+			}
+			// Return error for unsupported errors.
+			return err
+		} // Success.
+		formatConfigs[index] = formatXL
+		successCount++
+	}
+	// All `format.json` has been read successfully, previously completed.
+	if successCount == len(bootstrapDisks) {
+		// Return success.
+		return nil
+	}
+	// All disks are fresh, format.json will be written by initFormatXL()
+	if formatNotFoundCount == len(bootstrapDisks) {
+		return initFormatXL(bootstrapDisks)
+	}
+	// Validate format configs for consistency in JBOD and disks.
+	if err := checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	if referenceConfig == nil {
+		// This config will be used to update the drives missing format.json.
+		for _, formatConfig := range formatConfigs {
+			if formatConfig == nil {
+				continue
+			}
+			referenceConfig = formatConfig
+			break
+		}
+	}
+
+	uuidUsage := make([]struct {
+		uuid  string // Disk uuid
+		inUse bool   // indicates if the uuid is used by
+		// any disk
+	}, len(bootstrapDisks))
+
+	// Returns any unused drive UUID.
+	getUnusedUUID := func() string {
+		for index := range uuidUsage {
+			if !uuidUsage[index].inUse {
+				uuidUsage[index].inUse = true
+				return uuidUsage[index].uuid
+			}
+		}
+		return ""
+	}
+
+	// From reference config update UUID's not be in use.
+	for index, diskUUID := range referenceConfig.XL.JBOD {
+		uuidUsage[index].uuid = diskUUID
+		uuidUsage[index].inUse = false
+	}
+
+	// For all config formats validate if they are in use and
+	// update the uuidUsage values.
+	for _, config := range formatConfigs {
+		if config == nil {
+			continue
+		}
+		for index := range uuidUsage {
+			if config.XL.Disk == uuidUsage[index].uuid {
+				uuidUsage[index].inUse = true
+				break
+			}
+		}
+	}
+	// This section heals the format.json and updates the fresh disks
+	// by apply a new UUID for all the fresh disks.
+	for index, heal := range needHeal {
+		if !heal {
+			continue
+		}
+		config := &formatConfigV1{}
+		*config = *referenceConfig
+		config.XL.Disk = getUnusedUUID()
+		if config.XL.Disk == "" {
+			// getUnusedUUID() should have
+			// returned an unused uuid, it
+			// is an unexpected error.
+			return errUnexpected
+		}
+
+		formatBytes, err := json.Marshal(config)
+		if err != nil {
+			return err
+		}
+		// Fresh disk without format.json
+		_, _ = bootstrapDisks[index].AppendFile(minioMetaBucket, formatConfigFile, formatBytes)
+		// Ignore any error from AppendFile() as
+		// quorum might still be there to be operational.
+	}
+	return nil
+}
+
+// loadFormatXL - loads XL `format.json` and returns back properly
+// ordered storage slice based on `format.json`.
+func loadFormatXL(bootstrapDisks []StorageAPI) (disks []StorageAPI, err error) {
+	var unformattedDisksFoundCnt = 0
+	var diskNotFoundCount = 0
+	formatConfigs := make([]*formatConfigV1, len(bootstrapDisks))
+
+	// Try to load `format.json` bootstrap disks.
+	for index, disk := range bootstrapDisks {
+		var formatXL *formatConfigV1
+		formatXL, err = loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk {
+				unformattedDisksFoundCnt++
+				continue
+			} else if err == errDiskNotFound {
+				diskNotFoundCount++
+				continue
+			}
+			return nil, err
+		}
+		// Save valid formats.
+		formatConfigs[index] = formatXL
+	}
+
+	// If all disks indicate that 'format.json' is not available
+	// return 'errUnformattedDisk'.
+	if unformattedDisksFoundCnt == len(bootstrapDisks) {
+		return nil, errUnformattedDisk
+	} else if diskNotFoundCount == len(bootstrapDisks) {
+		return nil, errDiskNotFound
+	} else if diskNotFoundCount > len(bootstrapDisks)-(len(bootstrapDisks)/2+1) {
+		return nil, errXLReadQuorum
+	} else if unformattedDisksFoundCnt > len(bootstrapDisks)-(len(bootstrapDisks)/2+1) {
+		return nil, errXLReadQuorum
+	}
+
+	// Validate the format configs read are correct.
+	if err = checkFormatXL(formatConfigs); err != nil {
+		return nil, err
+	}
+	// Erasure code requires disks to be presented in the same order each time.
+	return reorderDisks(bootstrapDisks, formatConfigs)
+}
+
+// checkFormatXL - verifies if format.json format is intact.
+func checkFormatXL(formatConfigs []*formatConfigV1) error {
+	for _, formatXL := range formatConfigs {
+		if formatXL == nil {
+			continue
+		}
+		// Validate format version and format type.
+		if formatXL.Version != "1" {
+			return fmt.Errorf("Unsupported version of backend format [%s] found.", formatXL.Version)
+		}
+		if formatXL.Format != "xl" {
+			return fmt.Errorf("Unsupported backend format [%s] found.", formatXL.Format)
+		}
+		if formatXL.XL.Version != "1" {
+			return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version)
+		}
+		if len(formatConfigs) != len(formatXL.XL.JBOD) {
+			return fmt.Errorf("Number of disks %d did not match the backend format %d", len(formatConfigs), len(formatXL.XL.JBOD))
+		}
+	}
+	if err := checkJBODConsistency(formatConfigs); err != nil {
+		return err
+	}
+	return checkDisksConsistency(formatConfigs)
+}
+
+// initFormatXL - save XL format configuration on all disks.
+func initFormatXL(storageDisks []StorageAPI) (err error) {
+	var (
+		jbod             = make([]string, len(storageDisks))
+		formats          = make([]*formatConfigV1, len(storageDisks))
+		saveFormatErrCnt = 0
+	)
+	for index, disk := range storageDisks {
+		if err = disk.MakeVol(minioMetaBucket); err != nil {
+			if err != errVolumeExists {
+				saveFormatErrCnt++
+				// Check for write quorum.
+				if saveFormatErrCnt <= len(storageDisks)-(len(storageDisks)/2+3) {
+					continue
+				}
+				return errXLWriteQuorum
+			}
+		}
+		var u *uuid.UUID
+		u, err = uuid.New()
+		if err != nil {
+			saveFormatErrCnt++
+			// Check for write quorum.
+			if saveFormatErrCnt <= len(storageDisks)-(len(storageDisks)/2+3) {
+				continue
+			}
+			return err
+		}
+		formats[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    u.String(),
+			},
+		}
+		jbod[index] = formats[index].XL.Disk
+	}
+	for index, disk := range storageDisks {
+		formats[index].XL.JBOD = jbod
+		formatBytes, err := json.Marshal(formats[index])
+		if err != nil {
+			return err
+		}
+		n, err := disk.AppendFile(minioMetaBucket, formatConfigFile, formatBytes)
+		if err != nil {
+			return err
+		}
+		if n != int64(len(formatBytes)) {
+			return errUnexpected
+		}
+	}
+	return nil
+}

fs-v1-metadata.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"encoding/json"
+	"path"
+	"sort"
+)
+
+const (
+	fsMetaJSONFile = "fs.json"
+)
+
+// A fsMetaV1 represents a metadata header mapping keys to sets of values.
+type fsMetaV1 struct {
+	Version string `json:"version"`
+	Format  string `json:"format"`
+	Minio   struct {
+		Release string `json:"release"`
+	} `json:"minio"`
+	Parts []objectPartInfo `json:"parts,omitempty"`
+}
+
+// ObjectPartIndex - returns the index of matching object part number.
+func (m fsMetaV1) ObjectPartIndex(partNumber int) (partIndex int) {
+	for i, part := range m.Parts {
+		if partNumber == part.Number {
+			partIndex = i
+			return partIndex
+		}
+	}
+	return -1
+}
+
+// AddObjectPart - add a new object part in order.
+func (m *fsMetaV1) AddObjectPart(partNumber int, partName string, partETag string, partSize int64) {
+	partInfo := objectPartInfo{
+		Number: partNumber,
+		Name:   partName,
+		ETag:   partETag,
+		Size:   partSize,
+	}
+
+	// Update part info if it already exists.
+	for i, part := range m.Parts {
+		if partNumber == part.Number {
+			m.Parts[i] = partInfo
+			return
+		}
+	}
+
+	// Proceed to include new part info.
+	m.Parts = append(m.Parts, partInfo)
+
+	// Parts in fsMeta should be in sorted order by part number.
+	sort.Sort(byObjectPartNumber(m.Parts))
+}
+
+// readFSMetadata - returns the object metadata `fs.json` content.
+func (fs fsObjects) readFSMetadata(bucket, object string) (fsMeta fsMetaV1, err error) {
+	var buffer []byte
+	buffer, err = readAll(fs.storage, bucket, path.Join(object, fsMetaJSONFile))
+	if err != nil {
+		return fsMetaV1{}, err
+	}
+	err = json.Unmarshal(buffer, &fsMeta)
+	if err != nil {
+		return fsMetaV1{}, err
+	}
+	return fsMeta, nil
+}
+
+// newFSMetaV1 - initializes new fsMetaV1.
+func newFSMetaV1() (fsMeta fsMetaV1) {
+	fsMeta = fsMetaV1{}
+	fsMeta.Version = "1"
+	fsMeta.Format = "fs"
+	fsMeta.Minio.Release = minioReleaseTag
+	return fsMeta
+}
+
+// writeFSMetadata - writes `fs.json` metadata.
+func (fs fsObjects) writeFSMetadata(bucket, prefix string, fsMeta fsMetaV1) error {
+	metadataBytes, err := json.Marshal(fsMeta)
+	if err != nil {
+		return err
+	}
+	n, err := fs.storage.AppendFile(bucket, path.Join(prefix, fsMetaJSONFile), metadataBytes)
+	if err != nil {
+		return err
+	}
+	if n != int64(len(metadataBytes)) {
+		return errUnexpected
+	}
+	return nil
+}

fs-v1-multipart-common.go

@@ -0,0 +1,70 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"path"
+	"strings"
+)
+
+// Returns if the prefix is a multipart upload.
+func (fs fsObjects) isMultipartUpload(bucket, prefix string) bool {
+	_, err := fs.storage.StatFile(bucket, pathJoin(prefix, uploadsJSONFile))
+	return err == nil
+}
+
+// listUploadsInfo - list all uploads info.
+func (fs fsObjects) listUploadsInfo(prefixPath string) (uploads []uploadInfo, err error) {
+	splitPrefixes := strings.SplitN(prefixPath, "/", 3)
+	uploadIDs, err := readUploadsJSON(splitPrefixes[1], splitPrefixes[2], fs.storage)
+	if err != nil {
+		if err == errFileNotFound {
+			return []uploadInfo{}, nil
+		}
+		return nil, err
+	}
+	uploads = uploadIDs.Uploads
+	return uploads, nil
+}
+
+// Checks whether bucket exists.
+func (fs fsObjects) isBucketExist(bucket string) bool {
+	// Check whether bucket exists.
+	_, err := fs.storage.StatVol(bucket)
+	if err != nil {
+		if err == errVolumeNotFound {
+			return false
+		}
+		errorIf(err, "Stat failed on bucket "+bucket+".")
+		return false
+	}
+	return true
+}
+
+// isUploadIDExists - verify if a given uploadID exists and is valid.
+func (fs fsObjects) isUploadIDExists(bucket, object, uploadID string) bool {
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	_, err := fs.storage.StatFile(minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
+	if err != nil {
+		if err == errFileNotFound {
+			return false
+		}
+		errorIf(err, "Unable to access upload id"+uploadIDPath)
+		return false
+	}
+	return true
+}

fs-v1-multipart.go

@@ -0,0 +1,648 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/skyrings/skyring-common/tools/uuid"
+)
+
+// listMultipartUploads - lists all multipart uploads.
+func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+	result := ListMultipartsInfo{}
+	recursive := true
+	if delimiter == slashSeparator {
+		recursive = false
+	}
+
+	result.IsTruncated = true
+	result.MaxUploads = maxUploads
+	result.KeyMarker = keyMarker
+	result.Prefix = prefix
+	result.Delimiter = delimiter
+
+	// Not using path.Join() as it strips off the trailing '/'.
+	multipartPrefixPath := pathJoin(mpartMetaPrefix, bucket, prefix)
+	if prefix == "" {
+		// Should have a trailing "/" if prefix is ""
+		// For ex. multipartPrefixPath should be "multipart/bucket/" if prefix is ""
+		multipartPrefixPath += slashSeparator
+	}
+	multipartMarkerPath := ""
+	if keyMarker != "" {
+		multipartMarkerPath = pathJoin(mpartMetaPrefix, bucket, keyMarker)
+	}
+	var uploads []uploadMetadata
+	var err error
+	var eof bool
+	if uploadIDMarker != "" {
+		nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker))
+		uploads, _, err = listMultipartUploadIDs(bucket, keyMarker, uploadIDMarker, maxUploads, fs.storage)
+		nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker))
+		if err != nil {
+			return ListMultipartsInfo{}, err
+		}
+		maxUploads = maxUploads - len(uploads)
+	}
+	if maxUploads > 0 {
+		walker := fs.lookupTreeWalk(listParams{minioMetaBucket, recursive, multipartMarkerPath, multipartPrefixPath})
+		if walker == nil {
+			walker = fs.startTreeWalk(minioMetaBucket, multipartPrefixPath, multipartMarkerPath, recursive, func(bucket, object string) bool {
+				return fs.isMultipartUpload(bucket, object)
+			})
+		}
+		for maxUploads > 0 {
+			walkResult, ok := <-walker.ch
+			if !ok {
+				// Closed channel.
+				eof = true
+				break
+			}
+			// For any walk error return right away.
+			if walkResult.err != nil {
+				// File not found or Disk not found is a valid case.
+				if walkResult.err == errFileNotFound || walkResult.err == errDiskNotFound {
+					eof = true
+					break
+				}
+				return ListMultipartsInfo{}, err
+			}
+			entry := strings.TrimPrefix(walkResult.entry, retainSlash(pathJoin(mpartMetaPrefix, bucket)))
+			if strings.HasSuffix(walkResult.entry, slashSeparator) {
+				uploads = append(uploads, uploadMetadata{
+					Object: entry,
+				})
+				maxUploads--
+				if maxUploads == 0 {
+					if walkResult.end {
+						eof = true
+						break
+					}
+				}
+				continue
+			}
+			var tmpUploads []uploadMetadata
+			var end bool
+			uploadIDMarker = ""
+			nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry))
+			tmpUploads, end, err = listMultipartUploadIDs(bucket, entry, uploadIDMarker, maxUploads, fs.storage)
+			nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry))
+			if err != nil {
+				return ListMultipartsInfo{}, err
+			}
+			uploads = append(uploads, tmpUploads...)
+			maxUploads -= len(tmpUploads)
+			if walkResult.end && end {
+				eof = true
+				break
+			}
+		}
+	}
+	// Loop through all the received uploads fill in the multiparts result.
+	for _, upload := range uploads {
+		var objectName string
+		var uploadID string
+		if strings.HasSuffix(upload.Object, slashSeparator) {
+			// All directory entries are common prefixes.
+			uploadID = "" // Upload ids are empty for CommonPrefixes.
+			objectName = upload.Object
+			result.CommonPrefixes = append(result.CommonPrefixes, objectName)
+		} else {
+			uploadID = upload.UploadID
+			objectName = upload.Object
+			result.Uploads = append(result.Uploads, upload)
+		}
+		result.NextKeyMarker = objectName
+		result.NextUploadIDMarker = uploadID
+	}
+	result.IsTruncated = !eof
+	if !result.IsTruncated {
+		result.NextKeyMarker = ""
+		result.NextUploadIDMarker = ""
+	}
+	return result, nil
+}
+
+// ListMultipartUploads - lists all the pending multipart uploads on a
+// bucket. Additionally takes 'prefix, keyMarker, uploadIDmarker and a
+// delimiter' which allows us to list uploads match a particular
+// prefix or lexically starting from 'keyMarker' or delimiting the
+// output to get a directory like listing.
+//
+// Implements S3 compatible ListMultipartUploads API. The resulting
+// ListMultipartsInfo structure is unmarshalled directly into XML and
+// replied back to the client.
+func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+	// Validate input arguments.
+	if !IsValidBucketName(bucket) {
+		return ListMultipartsInfo{}, BucketNameInvalid{Bucket: bucket}
+	}
+	if !fs.isBucketExist(bucket) {
+		return ListMultipartsInfo{}, BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectPrefix(prefix) {
+		return ListMultipartsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix}
+	}
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != slashSeparator {
+		return ListMultipartsInfo{}, UnsupportedDelimiter{
+			Delimiter: delimiter,
+		}
+	}
+	// Verify if marker has prefix.
+	if keyMarker != "" && !strings.HasPrefix(keyMarker, prefix) {
+		return ListMultipartsInfo{}, InvalidMarkerPrefixCombination{
+			Marker: keyMarker,
+			Prefix: prefix,
+		}
+	}
+	if uploadIDMarker != "" {
+		if strings.HasSuffix(keyMarker, slashSeparator) {
+			return ListMultipartsInfo{}, InvalidUploadIDKeyCombination{
+				UploadIDMarker: uploadIDMarker,
+				KeyMarker:      keyMarker,
+			}
+		}
+		id, err := uuid.Parse(uploadIDMarker)
+		if err != nil {
+			return ListMultipartsInfo{}, err
+		}
+		if id.IsZero() {
+			return ListMultipartsInfo{}, MalformedUploadID{
+				UploadID: uploadIDMarker,
+			}
+		}
+	}
+	return fs.listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
+}
+
+// newMultipartUpload - wrapper for initializing a new multipart
+// request, returns back a unique upload id.
+//
+// Internally this function creates 'uploads.json' associated for the
+// incoming object at '.minio/multipart/bucket/object/uploads.json' on
+// all the disks. `uploads.json` carries metadata regarding on going
+// multipart operation on the object.
+func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[string]string) (uploadID string, err error) {
+	// No metadata is set, allocate a new one.
+	if meta == nil {
+		meta = make(map[string]string)
+	}
+
+	// Initialize `fs.json` values.
+	fsMeta := newFSMetaV1()
+
+	// This lock needs to be held for any changes to the directory contents of ".minio/multipart/object/"
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object))
+
+	uploadID = getUUID()
+	initiated := time.Now().UTC()
+	// Create 'uploads.json'
+	if err = writeUploadJSON(bucket, object, uploadID, initiated, fs.storage); err != nil {
+		return "", err
+	}
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	tempUploadIDPath := path.Join(tmpMetaPrefix, uploadID)
+	if err = fs.writeFSMetadata(minioMetaBucket, tempUploadIDPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, tempUploadIDPath)
+	}
+	err = fs.storage.RenameFile(minioMetaBucket, path.Join(tempUploadIDPath, fsMetaJSONFile), minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
+	if err != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, path.Join(tempUploadIDPath, fsMetaJSONFile)); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tempUploadIDPath)
+		}
+		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+	// Return success.
+	return uploadID, nil
+}
+
+// NewMultipartUpload - initialize a new multipart upload, returns a
+// unique id. The unique id returned here is of UUID form, for each
+// subsequent request each UUID is unique.
+//
+// Implements S3 compatible initiate multipart API.
+func (fs fsObjects) NewMultipartUpload(bucket, object string, meta map[string]string) (string, error) {
+	meta = make(map[string]string) // Reset the meta value, we are not going to save headers for fs.
+	// Verify if bucket name is valid.
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify whether the bucket exists.
+	if !fs.isBucketExist(bucket) {
+		return "", BucketNotFound{Bucket: bucket}
+	}
+	// Verify if object name is valid.
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+	return fs.newMultipartUpload(bucket, object, meta)
+}
+
+// PutObjectPart - reads incoming data until EOF for the part file on
+// an ongoing multipart transaction. Internally incoming data is
+// written to '.minio/tmp' location and safely renamed to
+// '.minio/multipart' for reach parts.
+func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify whether the bucket exists.
+	if !fs.isBucketExist(bucket) {
+		return "", BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+
+	nsMutex.RLock(minioMetaBucket, uploadIDPath)
+	// Just check if the uploadID exists to avoid copy if it doesn't.
+	uploadIDExists := fs.isUploadIDExists(bucket, object, uploadID)
+	nsMutex.RUnlock(minioMetaBucket, uploadIDPath)
+	if !uploadIDExists {
+		return "", InvalidUploadID{UploadID: uploadID}
+	}
+
+	// Hold write lock on the part so that there is no parallel upload on the part.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(partID)))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(partID)))
+
+	partSuffix := fmt.Sprintf("object%d", partID)
+	tmpPartPath := path.Join(tmpMetaPrefix, uploadID, partSuffix)
+
+	// Initialize md5 writer.
+	md5Writer := md5.New()
+
+	var buf = make([]byte, blockSizeV1)
+	for {
+		n, err := io.ReadFull(data, buf)
+		if err == io.EOF {
+			break
+		}
+		if err != nil && err != io.ErrUnexpectedEOF {
+			return "", toObjectErr(err, bucket, object)
+		}
+		// Update md5 writer.
+		md5Writer.Write(buf[:n])
+		m, err := fs.storage.AppendFile(minioMetaBucket, tmpPartPath, buf[:n])
+		if err != nil {
+			return "", toObjectErr(err, bucket, object)
+		}
+		if m != int64(len(buf[:n])) {
+			return "", toObjectErr(errUnexpected, bucket, object)
+		}
+	}
+
+	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
+	if md5Hex != "" {
+		if newMD5Hex != md5Hex {
+			return "", BadDigest{md5Hex, newMD5Hex}
+		}
+	}
+
+	// Hold write lock as we are updating fs.json
+	nsMutex.Lock(minioMetaBucket, uploadIDPath)
+	defer nsMutex.Unlock(minioMetaBucket, uploadIDPath)
+
+	// Just check if the uploadID exists to avoid copy if it doesn't.
+	if !fs.isUploadIDExists(bucket, object, uploadID) {
+		return "", InvalidUploadID{UploadID: uploadID}
+	}
+
+	fsMeta, err := fs.readFSMetadata(minioMetaBucket, uploadIDPath)
+	if err != nil {
+		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+	fsMeta.AddObjectPart(partID, partSuffix, newMD5Hex, size)
+
+	partPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, partSuffix)
+	err = fs.storage.RenameFile(minioMetaBucket, tmpPartPath, minioMetaBucket, partPath)
+	if err != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tmpPartPath); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tmpPartPath)
+		}
+		return "", toObjectErr(err, minioMetaBucket, partPath)
+	}
+	uploadIDPath = path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	tempUploadIDPath := path.Join(tmpMetaPrefix, uploadID)
+	if err = fs.writeFSMetadata(minioMetaBucket, tempUploadIDPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, tempUploadIDPath)
+	}
+	err = fs.storage.RenameFile(minioMetaBucket, path.Join(tempUploadIDPath, fsMetaJSONFile), minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
+	if err != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, path.Join(tempUploadIDPath, fsMetaJSONFile)); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tempUploadIDPath)
+		}
+		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+	return newMD5Hex, nil
+}
+
+// listObjectParts - wrapper scanning through
+// '.minio/multipart/bucket/object/UPLOADID'. Lists all the parts
+// saved inside '.minio/multipart/bucket/object/UPLOADID'.
+func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
+	result := ListPartsInfo{}
+
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	fsMeta, err := fs.readFSMetadata(minioMetaBucket, uploadIDPath)
+	if err != nil {
+		return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+	// Only parts with higher part numbers will be listed.
+	partIdx := fsMeta.ObjectPartIndex(partNumberMarker)
+	parts := fsMeta.Parts
+	if partIdx != -1 {
+		parts = fsMeta.Parts[partIdx+1:]
+	}
+	count := maxParts
+	for _, part := range parts {
+		var fi FileInfo
+		partNamePath := path.Join(mpartMetaPrefix, bucket, object, uploadID, part.Name)
+		fi, err = fs.storage.StatFile(minioMetaBucket, partNamePath)
+		if err != nil {
+			return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, partNamePath)
+		}
+		result.Parts = append(result.Parts, partInfo{
+			PartNumber:   part.Number,
+			ETag:         part.ETag,
+			LastModified: fi.ModTime,
+			Size:         fi.Size,
+		})
+		count--
+		if count == 0 {
+			break
+		}
+	}
+	// If listed entries are more than maxParts, we set IsTruncated as true.
+	if len(parts) > len(result.Parts) {
+		result.IsTruncated = true
+		// Make sure to fill next part number marker if IsTruncated is
+		// true for subsequent listing.
+		nextPartNumberMarker := result.Parts[len(result.Parts)-1].PartNumber
+		result.NextPartNumberMarker = nextPartNumberMarker
+	}
+	result.Bucket = bucket
+	result.Object = object
+	result.UploadID = uploadID
+	result.MaxParts = maxParts
+	return result, nil
+}
+
+// ListObjectParts - lists all previously uploaded parts for a given
+// object and uploadID.  Takes additional input of part-number-marker
+// to indicate where the listing should begin from.
+//
+// Implements S3 compatible ListObjectParts API. The resulting
+// ListPartsInfo structure is unmarshalled directly into XML and
+// replied back to the client.
+func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return ListPartsInfo{}, BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify whether the bucket exists.
+	if !fs.isBucketExist(bucket) {
+		return ListPartsInfo{}, BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return ListPartsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+	// Hold lock so that there is no competing abort-multipart-upload or complete-multipart-upload.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+
+	if !fs.isUploadIDExists(bucket, object, uploadID) {
+		return ListPartsInfo{}, InvalidUploadID{UploadID: uploadID}
+	}
+	return fs.listObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
+}
+
+// CompleteMultipartUpload - completes an ongoing multipart
+// transaction after receiving all the parts indicated by the client.
+// Returns an md5sum calculated by concatenating all the individual
+// md5sums of all the parts.
+//
+// Implements S3 compatible Complete multipart API.
+func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (string, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify whether the bucket exists.
+	if !fs.isBucketExist(bucket) {
+		return "", BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{
+			Bucket: bucket,
+			Object: object,
+		}
+	}
+
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	// Hold lock so that
+	// 1) no one aborts this multipart upload
+	// 2) no one does a parallel complete-multipart-upload on this
+	// multipart upload
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+
+	if !fs.isUploadIDExists(bucket, object, uploadID) {
+		return "", InvalidUploadID{UploadID: uploadID}
+	}
+
+	// Read saved fs metadata for ongoing multipart.
+	fsMeta, err := fs.readFSMetadata(minioMetaBucket, uploadIDPath)
+	if err != nil {
+		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+
+	// Calculate s3 compatible md5sum for complete multipart.
+	s3MD5, err := completeMultipartMD5(parts...)
+	if err != nil {
+		return "", err
+	}
+
+	tempObj := path.Join(tmpMetaPrefix, uploadID, "object1")
+	var buffer = make([]byte, blockSizeV1)
+
+	// Loop through all parts, validate them and then commit to disk.
+	for i, part := range parts {
+		partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
+		if partIdx == -1 {
+			return "", InvalidPart{}
+		}
+		if fsMeta.Parts[partIdx].ETag != part.ETag {
+			return "", BadDigest{}
+		}
+		// All parts except the last part has to be atleast 5MB.
+		if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
+			return "", PartTooSmall{}
+		}
+		// Construct part suffix.
+		partSuffix := fmt.Sprintf("object%d", part.PartNumber)
+		multipartPartFile := path.Join(mpartMetaPrefix, bucket, object, uploadID, partSuffix)
+		offset := int64(0)
+		totalLeft := fsMeta.Parts[partIdx].Size
+		for totalLeft > 0 {
+			var n int64
+			n, err = fs.storage.ReadFile(minioMetaBucket, multipartPartFile, offset, buffer)
+			if err != nil {
+				if err == errFileNotFound {
+					return "", InvalidPart{}
+				}
+				return "", toObjectErr(err, minioMetaBucket, multipartPartFile)
+			}
+			n, err = fs.storage.AppendFile(minioMetaBucket, tempObj, buffer[:n])
+			if err != nil {
+				return "", toObjectErr(err, minioMetaBucket, tempObj)
+			}
+			offset += n
+			totalLeft -= n
+		}
+	}
+
+	// Rename the file back to original location, if not delete the temporary object.
+	err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
+	if err != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempObj); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tempObj)
+		}
+		return "", toObjectErr(err, bucket, object)
+	}
+
+	// Cleanup all the parts if everything else has been safely committed.
+	if err = cleanupUploadedParts(bucket, object, uploadID, fs.storage); err != nil {
+		return "", err
+	}
+
+	// Hold the lock so that two parallel complete-multipart-uploads do not
+	// leave a stale uploads.json behind.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object))
+
+	// Validate if there are other incomplete upload-id's present for
+	// the object, if yes do not attempt to delete 'uploads.json'.
+	uploadsJSON, err := readUploadsJSON(bucket, object, fs.storage)
+	if err != nil {
+		return "", toObjectErr(err, minioMetaBucket, object)
+	}
+	// If we have successfully read `uploads.json`, then we proceed to
+	// purge or update `uploads.json`.
+	uploadIDIdx := uploadsJSON.Index(uploadID)
+	if uploadIDIdx != -1 {
+		uploadsJSON.Uploads = append(uploadsJSON.Uploads[:uploadIDIdx], uploadsJSON.Uploads[uploadIDIdx+1:]...)
+	}
+	if len(uploadsJSON.Uploads) > 0 {
+		if err = updateUploadsJSON(bucket, object, uploadsJSON, fs.storage); err != nil {
+			return "", toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object))
+		}
+		// Return success.
+		return s3MD5, nil
+	}
+
+	if err = fs.storage.DeleteFile(minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object))
+	}
+
+	// Return md5sum.
+	return s3MD5, nil
+}
+
+// abortMultipartUpload - wrapper for purging an ongoing multipart
+// transaction, deletes uploadID entry from `uploads.json` and purges
+// the directory at '.minio/multipart/bucket/object/uploadID' holding
+// all the upload parts.
+func (fs fsObjects) abortMultipartUpload(bucket, object, uploadID string) error {
+	// Cleanup all uploaded parts.
+	if err := cleanupUploadedParts(bucket, object, uploadID, fs.storage); err != nil {
+		return err
+	}
+
+	// Validate if there are other incomplete upload-id's present for
+	// the object, if yes do not attempt to delete 'uploads.json'.
+	uploadsJSON, err := readUploadsJSON(bucket, object, fs.storage)
+	if err == nil {
+		uploadIDIdx := uploadsJSON.Index(uploadID)
+		if uploadIDIdx != -1 {
+			uploadsJSON.Uploads = append(uploadsJSON.Uploads[:uploadIDIdx], uploadsJSON.Uploads[uploadIDIdx+1:]...)
+		}
+		// There are pending uploads for the same object, preserve
+		// them update 'uploads.json' in-place.
+		if len(uploadsJSON.Uploads) > 0 {
+			err = updateUploadsJSON(bucket, object, uploadsJSON, fs.storage)
+			if err != nil {
+				return toObjectErr(err, bucket, object)
+			}
+			return nil
+		}
+	} // No more pending uploads for the object, we purge the entire
+	// entry at '.minio/multipart/bucket/object'.
+	if err = fs.storage.DeleteFile(minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)); err != nil {
+		return toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object))
+	}
+	return nil
+}
+
+// AbortMultipartUpload - aborts an ongoing multipart operation
+// signified by the input uploadID. This is an atomic operation
+// doesn't require clients to initiate multiple such requests.
+//
+// All parts are purged from all disks and reference to the uploadID
+// would be removed from the system, rollback is not possible on this
+// operation.
+//
+// Implements S3 compatible Abort multipart API, slight difference is
+// that this is an atomic idempotent operation. Subsequent calls have
+// no affect and further requests to the same uploadID would not be
+// honored.
+func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	if !fs.isBucketExist(bucket) {
+		return BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+
+	// Hold lock so that there is no competing complete-multipart-upload or put-object-part.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+
+	if !fs.isUploadIDExists(bucket, object, uploadID) {
+		return InvalidUploadID{UploadID: uploadID}
+	}
+
+	err := fs.abortMultipartUpload(bucket, object, uploadID)
+	return err
+}

fs-v1.go

@@ -0,0 +1,435 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/mimedb"
+)
+
+// fsObjects - Implements fs object layer.
+type fsObjects struct {
+	storage            StorageAPI
+	physicalDisk       string
+	listObjectMap      map[listParams][]*treeWalkerFS
+	listObjectMapMutex *sync.Mutex
+}
+
+// newFSObjects - initialize new fs object layer.
+func newFSObjects(disk string) (ObjectLayer, error) {
+	storage, err := newStorageAPI(disk)
+	if err != nil {
+		return nil, err
+	}
+
+	// Runs house keeping code, like creating minioMetaBucket, cleaning up tmp files etc.
+	fsHouseKeeping(storage)
+
+	// Return successfully initialized object layer.
+	return fsObjects{
+		storage:            storage,
+		physicalDisk:       disk,
+		listObjectMap:      make(map[listParams][]*treeWalkerFS),
+		listObjectMapMutex: &sync.Mutex{},
+	}, nil
+}
+
+// StorageInfo - returns underlying storage statistics.
+func (fs fsObjects) StorageInfo() StorageInfo {
+	info, err := disk.GetInfo(fs.physicalDisk)
+	fatalIf(err, "Unable to get disk info "+fs.physicalDisk)
+	return StorageInfo{
+		Total: info.Total,
+		Free:  info.Free,
+	}
+}
+
+/// Bucket operations
+
+// MakeBucket - make a bucket.
+func (fs fsObjects) MakeBucket(bucket string) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	if err := fs.storage.MakeVol(bucket); err != nil {
+		return toObjectErr(err, bucket)
+	}
+	return nil
+}
+
+// GetBucketInfo - get bucket info.
+func (fs fsObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketInfo{}, BucketNameInvalid{Bucket: bucket}
+	}
+	vi, err := fs.storage.StatVol(bucket)
+	if err != nil {
+		return BucketInfo{}, toObjectErr(err, bucket)
+	}
+	return BucketInfo{
+		Name:    bucket,
+		Created: vi.Created,
+	}, nil
+}
+
+// ListBuckets - list buckets.
+func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
+	var bucketInfos []BucketInfo
+	vols, err := fs.storage.ListVols()
+	if err != nil {
+		return nil, toObjectErr(err)
+	}
+	for _, vol := range vols {
+		// StorageAPI can send volume names which are incompatible
+		// with buckets, handle it and skip them.
+		if !IsValidBucketName(vol.Name) {
+			continue
+		}
+		bucketInfos = append(bucketInfos, BucketInfo{
+			Name:    vol.Name,
+			Created: vol.Created,
+		})
+	}
+	sort.Sort(byBucketName(bucketInfos))
+	return bucketInfos, nil
+}
+
+// DeleteBucket - delete a bucket.
+func (fs fsObjects) DeleteBucket(bucket string) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	if err := fs.storage.DeleteVol(bucket); err != nil {
+		return toObjectErr(err, bucket)
+	}
+	return nil
+}
+
+/// Object Operations
+
+// GetObject - get an object.
+func (fs fsObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify if object is valid.
+	if !IsValidObjectName(object) {
+		return ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+	var totalLeft = length
+	for totalLeft > 0 {
+		// Figure out the right blockSize as it was encoded before.
+		var curBlockSize int64
+		if blockSizeV1 < totalLeft {
+			curBlockSize = blockSizeV1
+		} else {
+			curBlockSize = totalLeft
+		}
+		buf := make([]byte, curBlockSize)
+		n, err := fs.storage.ReadFile(bucket, object, startOffset, buf)
+		if err != nil {
+			return toObjectErr(err, bucket, object)
+		}
+		_, err = writer.Write(buf[:n])
+		if err != nil {
+			return toObjectErr(err, bucket, object)
+		}
+		totalLeft -= n
+		startOffset += n
+	}
+	return nil
+}
+
+// GetObjectInfo - get object info.
+func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return ObjectInfo{}, (BucketNameInvalid{Bucket: bucket})
+	}
+	// Verify if object is valid.
+	if !IsValidObjectName(object) {
+		return ObjectInfo{}, (ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+	fi, err := fs.storage.StatFile(bucket, object)
+	if err != nil {
+		return ObjectInfo{}, toObjectErr(err, bucket, object)
+	}
+	contentType := "application/octet-stream"
+	if objectExt := filepath.Ext(object); objectExt != "" {
+		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
+		if ok {
+			contentType = content.ContentType
+		}
+	}
+	return ObjectInfo{
+		Bucket:      bucket,
+		Name:        object,
+		ModTime:     fi.ModTime,
+		Size:        fi.Size,
+		IsDir:       fi.Mode.IsDir(),
+		ContentType: contentType,
+		MD5Sum:      "", // Read from metadata.
+	}, nil
+}
+
+// PutObject - create an object.
+func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{
+			Bucket: bucket,
+			Object: object,
+		}
+	}
+
+	uniqueID := getUUID()
+
+	// Temporary object.
+	tempObj := path.Join(tmpMetaPrefix, uniqueID)
+
+	// Initialize md5 writer.
+	md5Writer := md5.New()
+
+	if size == 0 {
+		// For size 0 we write a 0byte file.
+		_, err := fs.storage.AppendFile(minioMetaBucket, tempObj, []byte(""))
+		if err != nil {
+			return "", toObjectErr(err, bucket, object)
+		}
+	} else {
+		// Allocate buffer.
+		buf := make([]byte, blockSizeV1)
+		for {
+			n, rErr := data.Read(buf)
+			if rErr == io.EOF {
+				break
+			}
+			if rErr != nil {
+				return "", toObjectErr(rErr, bucket, object)
+			}
+			// Update md5 writer.
+			md5Writer.Write(buf[:n])
+			m, wErr := fs.storage.AppendFile(minioMetaBucket, tempObj, buf[:n])
+			if wErr != nil {
+				return "", toObjectErr(wErr, bucket, object)
+			}
+			if m != int64(len(buf[:n])) {
+				return "", toObjectErr(errUnexpected, bucket, object)
+			}
+		}
+	}
+
+	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
+	// md5Hex representation.
+	var md5Hex string
+	if len(metadata) != 0 {
+		md5Hex = metadata["md5Sum"]
+	}
+	if md5Hex != "" {
+		if newMD5Hex != md5Hex {
+			return "", BadDigest{md5Hex, newMD5Hex}
+		}
+	}
+
+	err := fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
+	if err != nil {
+		return "", toObjectErr(err, bucket, object)
+	}
+
+	// Return md5sum, successfully wrote object.
+	return newMD5Hex, nil
+}
+
+func (fs fsObjects) DeleteObject(bucket, object string) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+	if err := fs.storage.DeleteFile(bucket, object); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+	return nil
+}
+
+// Checks whether bucket exists.
+func isBucketExist(storage StorageAPI, bucketName string) bool {
+	// Check whether bucket exists.
+	_, err := storage.StatVol(bucketName)
+	if err != nil {
+		if err == errVolumeNotFound {
+			return false
+		}
+		errorIf(err, "Stat failed on bucket "+bucketName+".")
+		return false
+	}
+	return true
+}
+
+func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+	// Convert entry to FileInfo
+	entryToFileInfo := func(entry string) (fileInfo FileInfo, err error) {
+		if strings.HasSuffix(entry, slashSeparator) {
+			// Object name needs to be full path.
+			fileInfo.Name = entry
+			fileInfo.Mode = os.ModeDir
+			return
+		}
+		if fileInfo, err = fs.storage.StatFile(bucket, entry); err != nil {
+			return
+		}
+		// Object name needs to be full path.
+		fileInfo.Name = entry
+		return
+	}
+
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return ListObjectsInfo{}, BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify if bucket exists.
+	if !isBucketExist(fs.storage, bucket) {
+		return ListObjectsInfo{}, BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectPrefix(prefix) {
+		return ListObjectsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix}
+	}
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != slashSeparator {
+		return ListObjectsInfo{}, UnsupportedDelimiter{
+			Delimiter: delimiter,
+		}
+	}
+	// Verify if marker has prefix.
+	if marker != "" {
+		if !strings.HasPrefix(marker, prefix) {
+			return ListObjectsInfo{}, InvalidMarkerPrefixCombination{
+				Marker: marker,
+				Prefix: prefix,
+			}
+		}
+	}
+
+	// With max keys of zero we have reached eof, return right here.
+	if maxKeys == 0 {
+		return ListObjectsInfo{}, nil
+	}
+
+	// For delimiter and prefix as '/' we do not list anything at all
+	// since according to s3 spec we stop at the 'delimiter'
+	// along // with the prefix. On a flat namespace with 'prefix'
+	// as '/' we don't have any entries, since all the keys are
+	// of form 'keyName/...'
+	if delimiter == slashSeparator && prefix == slashSeparator {
+		return ListObjectsInfo{}, nil
+	}
+
+	// Over flowing count - reset to maxObjectList.
+	if maxKeys < 0 || maxKeys > maxObjectList {
+		maxKeys = maxObjectList
+	}
+
+	// Default is recursive, if delimiter is set then list non recursive.
+	recursive := true
+	if delimiter == slashSeparator {
+		recursive = false
+	}
+
+	walker := fs.lookupTreeWalk(listParams{bucket, recursive, marker, prefix})
+	if walker == nil {
+		walker = fs.startTreeWalk(bucket, prefix, marker, recursive, func(bucket, object string) bool {
+			return !strings.HasSuffix(object, slashSeparator)
+		})
+	}
+	var fileInfos []FileInfo
+	var eof bool
+	var nextMarker string
+	for i := 0; i < maxKeys; {
+		walkResult, ok := <-walker.ch
+		if !ok {
+			// Closed channel.
+			eof = true
+			break
+		}
+		// For any walk error return right away.
+		if walkResult.err != nil {
+			// File not found is a valid case.
+			if walkResult.err == errFileNotFound {
+				return ListObjectsInfo{}, nil
+			}
+			return ListObjectsInfo{}, toObjectErr(walkResult.err, bucket, prefix)
+		}
+		fileInfo, err := entryToFileInfo(walkResult.entry)
+		if err != nil {
+			return ListObjectsInfo{}, nil
+		}
+		nextMarker = fileInfo.Name
+		fileInfos = append(fileInfos, fileInfo)
+		if walkResult.end {
+			eof = true
+			break
+		}
+		i++
+	}
+	params := listParams{bucket, recursive, nextMarker, prefix}
+	if !eof {
+		fs.saveTreeWalk(params, walker)
+	}
+
+	result := ListObjectsInfo{IsTruncated: !eof}
+	for _, fileInfo := range fileInfos {
+		// With delimiter set we fill in NextMarker and Prefixes.
+		if delimiter == slashSeparator {
+			result.NextMarker = fileInfo.Name
+			if fileInfo.Mode.IsDir() {
+				result.Prefixes = append(result.Prefixes, fileInfo.Name)
+				continue
+			}
+		}
+		result.Objects = append(result.Objects, ObjectInfo{
+			Name:    fileInfo.Name,
+			ModTime: fileInfo.ModTime,
+			Size:    fileInfo.Size,
+			IsDir:   false,
+		})
+	}
+	return result, nil
+}
+
+// ListObjects - list all objects.
+func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+	return fs.listObjects(bucket, prefix, marker, delimiter, maxKeys)
+}

generic-handlers.go

@@ -17,7 +17,6 @@
 package main
 
 import (
-	"errors"
 	"net/http"
 	"path"
 	"regexp"
@@ -28,11 +27,6 @@ import (
 	"github.com/rs/cors"
 )
 
-const (
-	iso8601Format = "20060102T150405Z"
-	privateBucket = "/minio"
-)
-
 // HandlerFunc - useful to chain different middleware http.Handler
 type HandlerFunc func(http.Handler) http.Handler
 
@@ -51,18 +45,23 @@ type redirectHandler struct {
 	locationPrefix string
 }
 
+// Reserved bucket.
+const (
+	reservedBucket = "/minio"
+)
+
 func setBrowserRedirectHandler(h http.Handler) http.Handler {
-	return redirectHandler{handler: h, locationPrefix: privateBucket}
+	return redirectHandler{handler: h, locationPrefix: reservedBucket}
 }
 
 func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Re-direction handled specifically for browsers.
 	if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
 		// '/' is redirected to 'locationPrefix/'
-		// '/rpc' is redirected to 'locationPrefix/rpc'
+		// '/webrpc' is redirected to 'locationPrefix/webrpc'
 		// '/login' is redirected to 'locationPrefix/login'
 		switch r.URL.Path {
-		case "/", "/rpc", "/login", "/favicon.ico":
+		case "/", "/webrpc", "/login", "/favicon.ico":
 			location := h.locationPrefix + r.URL.Path
 			// Redirect to new location.
 			http.Redirect(w, r, location, http.StatusTemporaryRedirect)
@@ -84,7 +83,7 @@ func setBrowserCacheControlHandler(h http.Handler) http.Handler {
 func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if r.Method == "GET" && strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
 		// For all browser requests set appropriate Cache-Control policies
-		match, e := regexp.MatchString(privateBucket+`/([^/]+\.js|favicon.ico)`, r.URL.Path)
+		match, e := regexp.MatchString(reservedBucket+`/([^/]+\.js|favicon.ico)`, r.URL.Path)
 		if e != nil {
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 			return
@@ -93,7 +92,7 @@ func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			// For assets set cache expiry of one year. For each release, the name
 			// of the asset name will change and hence it can not be served from cache.
 			w.Header().Set("Cache-Control", "max-age=31536000")
-		} else if strings.HasPrefix(r.URL.Path, privateBucket+"/") {
+		} else if strings.HasPrefix(r.URL.Path, reservedBucket+"/") {
 			// For non asset requests we serve index.html which will never be cached.
 			w.Header().Set("Cache-Control", "no-store")
 		}
@@ -103,68 +102,60 @@ func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 // Adds verification for incoming paths.
 type minioPrivateBucketHandler struct {
-	handler       http.Handler
-	privateBucket string
+	handler        http.Handler
+	reservedBucket string
 }
 
 func setPrivateBucketHandler(h http.Handler) http.Handler {
-	return minioPrivateBucketHandler{handler: h, privateBucket: privateBucket}
+	return minioPrivateBucketHandler{handler: h, reservedBucket: reservedBucket}
 }
 
 func (h minioPrivateBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// For all non browser requests, reject access to 'privateBucket'.
-	if !strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && path.Clean(r.URL.Path) == privateBucket {
+	// For all non browser requests, reject access to 'reservedBucket'.
+	if !strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && path.Clean(r.URL.Path) == reservedBucket {
 		writeErrorResponse(w, r, ErrAllAccessDisabled, r.URL.Path)
 		return
 	}
 	h.handler.ServeHTTP(w, r)
 }
 
-// Supported incoming date formats.
-var timeFormats = []string{
+// Supported Amz date formats.
+var amzDateFormats = []string{
 	time.RFC1123,
 	time.RFC1123Z,
 	iso8601Format,
+	// Add new AMZ date formats here.
 }
 
-// Attempts to parse date string into known date layouts. Date layouts
-// currently supported are
-//  - ``time.RFC1123``
-//  - ``time.RFC1123Z``
-//  - ``iso8601Format``
-func parseDate(date string) (parsedTime time.Time, e error) {
-	for _, layout := range timeFormats {
-		parsedTime, e = time.Parse(layout, date)
+// parseAmzDate - parses date string into supported amz date formats.
+func parseAmzDate(amzDateStr string) (amzDate time.Time, apiErr APIErrorCode) {
+	for _, dateFormat := range amzDateFormats {
+		amzDate, e := time.Parse(dateFormat, amzDateStr)
 		if e == nil {
-			return parsedTime, nil
+			return amzDate, ErrNone
 		}
 	}
-	return time.Time{}, e
+	return time.Time{}, ErrMalformedDate
 }
 
-// Parse date string from incoming header, current supports and verifies
-// follow HTTP headers.
-//
-//  - X-Amz-Date
-//  - X-Minio-Date
-//  - Date
-//
-// In following time layouts ``time.RFC1123``, ``time.RFC1123Z`` and
-// ``iso8601Format``.
-var dateHeaders = []string{
+// Supported Amz date headers.
+var amzDateHeaders = []string{
 	"x-amz-date",
 	"x-minio-date",
 	"date",
 }
 
-func parseDateHeader(req *http.Request) (time.Time, error) {
-	for _, dateHeader := range dateHeaders {
-		date := req.Header.Get(http.CanonicalHeaderKey(dateHeader))
-		if date != "" {
-			return parseDate(date)
+// parseAmzDateHeader - parses supported amz date headers, in
+// supported amz date formats.
+func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
+	for _, amzDateHeader := range amzDateHeaders {
+		amzDateStr := req.Header.Get(http.CanonicalHeaderKey(amzDateHeader))
+		if amzDateStr != "" {
+			return parseAmzDate(amzDateStr)
 		}
 	}
-	return time.Time{}, errors.New("Date header missing, invalid request.")
+	// Date header missing.
+	return time.Time{}, ErrMissingDateHeader
 }
 
 type timeHandler struct {
@@ -178,18 +169,18 @@ func setTimeValidityHandler(h http.Handler) http.Handler {
 
 func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Verify if date headers are set, if not reject the request
-	if r.Header.Get("Authorization") != "" {
-		date, e := parseDateHeader(r)
-		if e != nil {
+	if _, ok := r.Header["Authorization"]; ok {
+		amzDate, apiErr := parseAmzDateHeader(r)
+		if apiErr != ErrNone {
 			// All our internal APIs are sensitive towards Date
 			// header, for all requests where Date header is not
 			// present we will reject such clients.
-			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
+			writeErrorResponse(w, r, apiErr, r.URL.Path)
 			return
 		}
 		// Verify if the request date header is more than 5minutes
 		// late, reject such clients.
-		if time.Now().UTC().Sub(date)/time.Minute > time.Duration(5)*time.Minute {
+		if time.Now().UTC().Sub(amzDate)/time.Minute > time.Duration(5)*time.Minute {
 			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
 			return
 		}
@@ -207,6 +198,7 @@ func setCorsHandler(h http.Handler) http.Handler {
 		AllowedOrigins: []string{"*"},
 		AllowedMethods: []string{"GET", "HEAD", "POST", "PUT"},
 		AllowedHeaders: []string{"*"},
+		ExposedHeaders: []string{"ETag"},
 	})
 	return c.Handler(h)
 }

globals.go

@@ -16,7 +16,34 @@
 
 package main
 
+import "github.com/fatih/color"
+
 // Global constants for Minio.
 const (
 	minGoVersion = ">= 1.6" // Minio requires at least Go v1.6
 )
+
+// minio configuration related constants.
+const (
+	globalMinioConfigVersion = "4"
+	globalMinioConfigDir     = ".minio"
+	globalMinioCertsDir      = ".minio/certs"
+	globalMinioCertFile      = "public.crt"
+	globalMinioKeyFile       = "private.key"
+	globalMinioConfigFile    = "config.json"
+	globalMinioProfilePath   = "profile"
+	// Add new global values here.
+)
+
+var (
+	globalQuiet = false // Quiet flag set via command line
+	globalTrace = false // Trace flag set via environment setting.
+	// Add new global flags here.
+)
+
+// global colors.
+var (
+	colorMagenta = color.New(color.FgMagenta, color.Bold).SprintfFunc()
+	colorWhite   = color.New(color.FgWhite, color.Bold).SprintfFunc()
+	colorGreen   = color.New(color.FgGreen, color.Bold).SprintfFunc()
+)

handler-utils.go

@@ -0,0 +1,54 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io"
+)
+
+// validates location constraint from the request body.
+// the location value in the request body should match the Region in serverConfig.
+// other values of location are not accepted.
+// make bucket fails in such cases.
+func isValidLocationContraint(reqBody io.Reader, serverRegion string) APIErrorCode {
+	var locationContraint createBucketLocationConfiguration
+	var errCode APIErrorCode
+	errCode = ErrNone
+	e := xmlDecoder(reqBody, &locationContraint)
+	if e != nil {
+		if e == io.EOF {
+			// Do nothing.
+			// failed due to empty body. The location will be set to default value from the serverConfig.
+			// this is valid.
+			errCode = ErrNone
+		} else {
+			// Failed due to malformed configuration.
+			errCode = ErrMalformedXML
+			//writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
+		}
+	} else {
+		// Region obtained from the body.
+		// It should be equal to Region in serverConfig.
+		// Else ErrInvalidRegion returned.
+		// For empty value location will be to set to  default value from the serverConfig.
+		if locationContraint.Location != "" && serverRegion != locationContraint.Location {
+			//writeErrorResponse(w, r, ErrInvalidRegion, r.URL.Path)
+			errCode = ErrInvalidRegion
+		}
+	}
+	return errCode
+}

handler-utils_test.go

@@ -0,0 +1,66 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"encoding/xml"
+	"io/ioutil"
+	"net/http"
+	"testing"
+)
+
+// Tests validate bucket LocationConstraint.
+func TestIsValidLocationContraint(t *testing.T) {
+	// generates the input request with XML bucket configuration set to the request body.
+	createExpectedRequest := func(req *http.Request, location string) (*http.Request, error) {
+		createBucketConfig := createBucketLocationConfiguration{}
+		createBucketConfig.Location = location
+		var createBucketConfigBytes []byte
+		createBucketConfigBytes, e := xml.Marshal(createBucketConfig)
+		if e != nil {
+			return nil, e
+		}
+		createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes)
+		req.Body = ioutil.NopCloser(createBucketConfigBuffer)
+		return req, nil
+	}
+
+	testCases := []struct {
+		locationForInputRequest string
+		serverConfigRegion      string
+		expectedCode            APIErrorCode
+	}{
+		// Test case - 1.
+		{"us-east-1", "us-east-1", ErrNone},
+		// Test case - 2.
+		// In case of empty request body ErrNone is returned.
+		{"", "us-east-1", ErrNone},
+		// Test case - 3.
+		{"eu-central-1", "us-east-1", ErrInvalidRegion},
+	}
+	for i, testCase := range testCases {
+		inputRequest, e := createExpectedRequest(&http.Request{}, testCase.locationForInputRequest)
+		if e != nil {
+			t.Fatalf("Test %d: Failed to Marshal bucket configuration", i+1)
+		}
+		actualCode := isValidLocationContraint(inputRequest.Body, testCase.serverConfigRegion)
+		if testCase.expectedCode != actualCode {
+			t.Errorf("Test %d: Expected the APIErrCode to be %d, but instead found %d", i+1, testCase.expectedCode, actualCode)
+		}
+	}
+}

httprange.go

@@ -21,15 +21,19 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-
-	"github.com/minio/minio/pkg/fs"
-	"github.com/minio/minio/pkg/probe"
 )
 
 const (
 	b = "bytes="
 )
 
+// InvalidRange - invalid range
+type InvalidRange struct{}
+
+func (e InvalidRange) Error() string {
+	return "The requested range is not satisfiable"
+}
+
 // HttpRange specifies the byte range to be sent to the client.
 type httpRange struct {
 	start, length, size int64
@@ -41,7 +45,7 @@ func (r *httpRange) String() string {
 }
 
 // Grab new range from request header
-func getRequestedRange(hrange string, size int64) (*httpRange, *probe.Error) {
+func getRequestedRange(hrange string, size int64) (*httpRange, error) {
 	r := &httpRange{
 		start:  0,
 		length: 0,
@@ -51,16 +55,16 @@ func getRequestedRange(hrange string, size int64) (*httpRange, *probe.Error) {
 	if hrange != "" {
 		err := r.parseRange(hrange)
 		if err != nil {
-			return nil, err.Trace()
+			return nil, err
 		}
 	}
 	return r, nil
 }
 
-func (r *httpRange) parse(ra string) *probe.Error {
+func (r *httpRange) parse(ra string) error {
 	i := strings.Index(ra, "-")
 	if i < 0 {
-		return probe.NewError(fs.InvalidRange{})
+		return InvalidRange{}
 	}
 	start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:])
 	if start == "" {
@@ -68,7 +72,7 @@ func (r *httpRange) parse(ra string) *probe.Error {
 		// range start relative to the end of the file.
 		i, err := strconv.ParseInt(end, 10, 64)
 		if err != nil {
-			return probe.NewError(fs.InvalidRange{})
+			return InvalidRange{}
 		}
 		if i > r.size {
 			i = r.size
@@ -78,7 +82,7 @@ func (r *httpRange) parse(ra string) *probe.Error {
 	} else {
 		i, err := strconv.ParseInt(start, 10, 64)
 		if err != nil || i > r.size || i < 0 {
-			return probe.NewError(fs.InvalidRange{})
+			return InvalidRange{}
 		}
 		r.start = i
 		if end == "" {
@@ -87,7 +91,7 @@ func (r *httpRange) parse(ra string) *probe.Error {
 		} else {
 			i, err := strconv.ParseInt(end, 10, 64)
 			if err != nil || r.start > i {
-				return probe.NewError(fs.InvalidRange{})
+				return InvalidRange{}
 			}
 			if i >= r.size {
 				i = r.size - 1
@@ -99,26 +103,26 @@ func (r *httpRange) parse(ra string) *probe.Error {
 }
 
 // parseRange parses a Range header string as per RFC 2616.
-func (r *httpRange) parseRange(s string) *probe.Error {
+func (r *httpRange) parseRange(s string) error {
 	if s == "" {
-		return probe.NewError(errors.New("header not present"))
+		return errors.New("header not present")
 	}
 	if !strings.HasPrefix(s, b) {
-		return probe.NewError(fs.InvalidRange{})
+		return InvalidRange{}
 	}
 
 	ras := strings.Split(s[len(b):], ",")
 	if len(ras) == 0 {
-		return probe.NewError(errors.New("invalid request"))
+		return errors.New("invalid request")
 	}
 	// Just pick the first one and ignore the rest, we only support one range per object
 	if len(ras) > 1 {
-		return probe.NewError(errors.New("multiple ranges specified"))
+		return errors.New("multiple ranges specified")
 	}
 
 	ra := strings.TrimSpace(ras[0])
 	if ra == "" {
-		return probe.NewError(fs.InvalidRange{})
+		return InvalidRange{}
 	}
 	return r.parse(ra)
 }

logger-console-hook.go

@@ -0,0 +1,46 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// consoleLogger - default logger if not other logging is enabled.
+type consoleLogger struct {
+	Enable bool   `json:"enable"`
+	Level  string `json:"level"`
+}
+
+// enable console logger.
+func enableConsoleLogger() {
+	clogger := serverConfig.GetConsoleLogger()
+	if !clogger.Enable {
+		// Disable console logger if asked for.
+		log.Out = ioutil.Discard
+		return
+	}
+
+	// log.Out and log.Formatter use the default versions.
+	// Only set specific log level.
+	lvl, err := logrus.ParseLevel(clogger.Level)
+	fatalIf(err, "Unknown log level found in the config file.")
+
+	log.Level = lvl
+}

logger-file-hook.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,32 +21,39 @@ import (
 	"os"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio/pkg/probe"
 )
 
+type fileLogger struct {
+	Enable   bool   `json:"enable"`
+	Filename string `json:"fileName"`
+	Level    string `json:"level"`
+}
+
 type localFile struct {
 	*os.File
 }
 
-func log2File(filename string) *probe.Error {
-	fileHook, e := newFile(filename)
-	if e != nil {
-		return probe.NewError(e)
+func enableFileLogger() {
+	flogger := serverConfig.GetFileLogger()
+	if !flogger.Enable || flogger.Filename == "" {
+		return
 	}
-	log.Hooks.Add(fileHook)                 // Add a local file hook.
-	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
-	log.Level = logrus.InfoLevel            // Minimum log level.
-	return nil
-}
-
-func newFile(filename string) (*localFile, error) {
-	file, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
-	if err != nil {
-		return nil, err
-	}
-	return &localFile{file}, nil
+
+	file, err := os.OpenFile(flogger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	fatalIf(err, "Unable to open log file.")
+
+	// Add a local file hook.
+	log.Hooks.Add(&localFile{file})
+
+	lvl, err := logrus.ParseLevel(flogger.Level)
+	fatalIf(err, "Unknown log level found in the config file.")
+
+	// Set default JSON formatter.
+	log.Formatter = new(logrus.JSONFormatter)
+	log.Level = lvl // Minimum log level.
 }
 
+// Fire fires the file logger hook and logs to the file.
 func (l *localFile) Fire(entry *logrus.Entry) error {
 	line, err := entry.String()
 	if err != nil {
@@ -57,14 +64,11 @@ func (l *localFile) Fire(entry *logrus.Entry) error {
 	return nil
 }
 
-// Levels -
+// Levels - indicate log levels supported.
 func (l *localFile) Levels() []logrus.Level {
 	return []logrus.Level{
 		logrus.PanicLevel,
 		logrus.FatalLevel,
 		logrus.ErrorLevel,
-		logrus.WarnLevel,
-		logrus.InfoLevel,
-		logrus.DebugLevel,
 	}
 }

logger-mongo-hook.go

@@ -1,75 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"fmt"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio/pkg/probe"
-	"gopkg.in/mgo.v2"
-	"gopkg.in/mgo.v2/bson"
-)
-
-// mongoDB collection
-type mongoDB struct {
-	c *mgo.Collection
-}
-
-func log2Mongo(url, db, collection string) *probe.Error {
-	mongoHook, e := newMongo(url, db, collection)
-	if e != nil {
-		return probe.NewError(e)
-	}
-	log.Hooks.Add(mongoHook)                // Add mongodb hook.
-	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
-	log.Level = logrus.InfoLevel            // Minimum log level.
-	return nil
-}
-
-// newMongo -
-func newMongo(mgoEndpoint, db, collection string) (*mongoDB, error) {
-	session, err := mgo.Dial(mgoEndpoint)
-	if err != nil {
-		return nil, err
-	}
-	return &mongoDB{c: session.DB(db).C(collection)}, nil
-}
-
-// Fire - the log event
-func (h *mongoDB) Fire(entry *logrus.Entry) error {
-	entry.Data["Level"] = entry.Level.String()
-	entry.Data["Time"] = entry.Time
-	entry.Data["Message"] = entry.Message
-	mgoErr := h.c.Insert(bson.M(entry.Data))
-	if mgoErr != nil {
-		return fmt.Errorf("Failed to send log entry to mongodb: %s", mgoErr)
-	}
-	return nil
-}
-
-// Levels -
-func (h *mongoDB) Levels() []logrus.Level {
-	return []logrus.Level{
-		logrus.PanicLevel,
-		logrus.FatalLevel,
-		logrus.ErrorLevel,
-		logrus.WarnLevel,
-		logrus.InfoLevel,
-		logrus.DebugLevel,
-	}
-}

logger-syslog-hook_nix.go

@@ -1,7 +1,7 @@
 // +build !windows
 
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,9 +23,14 @@ import (
 	"log/syslog"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio/pkg/probe"
 )
 
+type syslogLogger struct {
+	Enable bool   `json:"enable"`
+	Addr   string `json:"address"`
+	Level  string `json:"level"`
+}
+
 // syslogHook to send logs via syslog.
 type syslogHook struct {
 	writer        *syslog.Writer
@@ -33,15 +38,14 @@ type syslogHook struct {
 	syslogRaddr   string
 }
 
-func log2Syslog(network, raddr string) *probe.Error {
-	syslogHook, e := newSyslog(network, raddr, syslog.LOG_ERR, "MINIO")
-	if e != nil {
-		return probe.NewError(e)
-	}
+// enableSyslogLogger - enable logger at raddr.
+func enableSyslogLogger(raddr string) {
+	syslogHook, err := newSyslog("udp", raddr, syslog.LOG_ERR, "MINIO")
+	fatalIf(err, "Unable to initialize syslog logger.")
+
 	log.Hooks.Add(syslogHook)               // Add syslog hook.
 	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
-	log.Level = logrus.InfoLevel            // Minimum log level.
-	return nil
+	log.Level = logrus.ErrorLevel           // Minimum log level.
 }
 
 // newSyslog - Creates a hook to be added to an instance of logger.
@@ -63,12 +67,6 @@ func (hook *syslogHook) Fire(entry *logrus.Entry) error {
 		return hook.writer.Crit(line)
 	case logrus.ErrorLevel:
 		return hook.writer.Err(line)
-	case logrus.WarnLevel:
-		return hook.writer.Warning(line)
-	case logrus.InfoLevel:
-		return hook.writer.Info(line)
-	case logrus.DebugLevel:
-		return hook.writer.Debug(line)
 	default:
 		return nil
 	}
@@ -80,8 +78,5 @@ func (hook *syslogHook) Levels() []logrus.Level {
 		logrus.PanicLevel,
 		logrus.FatalLevel,
 		logrus.ErrorLevel,
-		logrus.WarnLevel,
-		logrus.InfoLevel,
-		logrus.DebugLevel,
 	}
 }

logger-syslog-hook_windows.go

@@ -1,7 +1,7 @@
 // +build windows
 
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,8 +18,13 @@
 
 package main
 
-import "github.com/minio/minio/pkg/probe"
-
-func log2Syslog(network, raddr string) *probe.Error {
-	return probe.NewError(errSysLogNotSupported)
+type syslogLogger struct {
+	Enable bool   `json:"enable"`
+	Addr   string `json:"address"`
+	Level  string `json:"level"`
+}
+
+// enableSyslogLogger - unsupported on windows.
+func enableSyslogLogger(raddr string) {
+	fatalIf(errSyslogNotSupported, "Unable to enable syslog.")
 }

logger.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,49 +17,105 @@
 package main
 
 import (
-	"encoding/json"
+	"bufio"
+	"bytes"
+	"os"
 	"reflect"
+	"runtime"
+	"runtime/debug"
+	"strconv"
+	"strings"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio/pkg/probe"
+	"github.com/dustin/go-humanize"
 )
 
 type fields map[string]interface{}
 
 var log = logrus.New() // Default console logger.
 
-func errorIf(err *probe.Error, msg string, fields map[string]interface{}) {
+// logger carries logging configuration for various supported loggers.
+// Currently supported loggers are
+//
+//   - console [default]
+//   - file
+//   - syslog
+//
+type logger struct {
+	Console consoleLogger `json:"console"`
+	File    fileLogger    `json:"file"`
+	Syslog  syslogLogger  `json:"syslog"`
+	// Add new loggers here.
+}
+
+// sysInfo returns useful system statistics.
+func sysInfo() map[string]string {
+	host, err := os.Hostname()
+	if err != nil {
+		host = ""
+	}
+	memstats := &runtime.MemStats{}
+	runtime.ReadMemStats(memstats)
+	return map[string]string{
+		"host.name":      host,
+		"host.os":        runtime.GOOS,
+		"host.arch":      runtime.GOARCH,
+		"host.lang":      runtime.Version(),
+		"host.cpus":      strconv.Itoa(runtime.NumCPU()),
+		"mem.used":       humanize.Bytes(memstats.Alloc),
+		"mem.total":      humanize.Bytes(memstats.Sys),
+		"mem.heap.used":  humanize.Bytes(memstats.HeapAlloc),
+		"mem.heap.total": humanize.Bytes(memstats.HeapSys),
+	}
+}
+
+// stackInfo returns printable stack trace.
+func stackInfo() string {
+	// Convert stack-trace bytes to io.Reader.
+	rawStack := bufio.NewReader(bytes.NewBuffer(debug.Stack()))
+	// Skip stack trace lines until our real caller.
+	for i := 0; i <= 4; i++ {
+		rawStack.ReadLine()
+	}
+
+	// Read the rest of useful stack trace.
+	stackBuf := new(bytes.Buffer)
+	stackBuf.ReadFrom(rawStack)
+
+	// Strip GOPATH of the build system and return.
+	return strings.Replace(stackBuf.String(), minioGOPATH+"/src/", "", -1)
+}
+
+// errorIf synonymous with fatalIf but doesn't exit on error != nil
+func errorIf(err error, msg string, data ...interface{}) {
 	if err == nil {
 		return
 	}
-	if fields == nil {
-		fields = make(map[string]interface{})
+	sysInfo := sysInfo()
+	fields := logrus.Fields{
+		"cause":   err.Error(),
+		"type":    reflect.TypeOf(err),
+		"sysInfo": sysInfo,
 	}
-	fields["Error"] = struct {
-		Cause     string             `json:"cause,omitempty"`
-		Type      string             `json:"type,omitempty"`
-		CallTrace []probe.TracePoint `json:"trace,omitempty"`
-		SysInfo   map[string]string  `json:"sysinfo,omitempty"`
-	}{
-		err.Cause.Error(),
-		reflect.TypeOf(err.Cause).String(),
-		err.CallTrace,
-		err.SysInfo,
+	if globalTrace {
+		fields["stack"] = "\n" + stackInfo()
 	}
-	log.WithFields(fields).Error(msg)
+	log.WithFields(fields).Errorf(msg, data...)
 }
 
-func fatalIf(err *probe.Error, msg string, fields map[string]interface{}) {
+// fatalIf wrapper function which takes error and prints jsonic error messages.
+func fatalIf(err error, msg string, data ...interface{}) {
 	if err == nil {
 		return
 	}
-	if fields == nil {
-		fields = make(map[string]interface{})
+	sysInfo := sysInfo()
+	fields := logrus.Fields{
+		"cause":   err.Error(),
+		"type":    reflect.TypeOf(err),
+		"sysInfo": sysInfo,
 	}
-
-	fields["error"] = err.ToGoError()
-	if jsonErr, e := json.Marshal(err); e == nil {
-		fields["probe"] = string(jsonErr)
+	if globalTrace {
+		fields["stack"] = "\n" + stackInfo()
 	}
-	log.WithFields(fields).Fatal(msg)
+	log.WithFields(fields).Fatalf(msg, data...)
 }

logger_test.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio/pkg/probe"
 
 	. "gopkg.in/check.v1"
 )
@@ -37,12 +36,12 @@ func (s *LoggerSuite) TestLogger(c *C) {
 	log.Out = &buffer
 	log.Formatter = new(logrus.JSONFormatter)
 
-	errorIf(probe.NewError(errors.New("Fake error")), "Failed with error.", nil)
+	errorIf(errors.New("Fake error"), "Failed with error.")
 	err := json.Unmarshal(buffer.Bytes(), &fields)
 	c.Assert(err, IsNil)
 	c.Assert(fields["level"], Equals, "error")
 
-	msg, ok := fields["Error"]
+	msg, ok := fields["cause"]
 	c.Assert(ok, Equals, true)
-	c.Assert(msg.(map[string]interface{})["cause"], Equals, "Fake error")
+	c.Assert(msg, Equals, "Fake error")
 }

main.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,14 +19,22 @@ package main
 import (
 	"fmt"
 	"os"
-	"runtime"
+	"path/filepath"
 	"sort"
-	"strconv"
 
-	"github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	"github.com/minio/mc/pkg/console"
-	"github.com/minio/minio/pkg/probe"
+	"github.com/pkg/profile"
+)
+
+var (
+	// global flags for minio.
+	minioFlags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "help, h",
+			Usage: "Show help.",
+		},
+	}
 )
 
 // Help template for minio.
@@ -47,53 +55,36 @@ FLAGS:
   {{end}}{{end}}
 VERSION:
   ` + minioVersion +
-	`{{ "\n"}}{{range $key, $value := ExtraInfo}}
-{{$key}}:
-  {{$value}}
-{{end}}`
+	`{{ "\n"}}`
 
 // init - check the environment before main starts
 func init() {
 	// Check if minio was compiled using a supported version of Golang.
 	checkGoVersion()
 
+	// Set global trace flag.
+	globalTrace = os.Getenv("MINIO_TRACE") == "1"
+
 	// It is an unsafe practice to run network services as
 	// root. Containers are an exception.
 	if !isContainerized() && os.Geteuid() == 0 {
 		console.Fatalln("Please run ‘minio’ as a non-root user.")
 	}
-
 }
 
 func migrate() {
 	// Migrate config file
 	migrateConfig()
+
+	// Migrate other configs here.
 }
 
-// Tries to get os/arch/platform specific information
-// Returns a map of current os/arch/platform/memstats
-func getSystemData() map[string]string {
-	host, err := os.Hostname()
-	if err != nil {
-		host = ""
-	}
-	memstats := &runtime.MemStats{}
-	runtime.ReadMemStats(memstats)
-	mem := fmt.Sprintf("Used: %s | Allocated: %s | Used-Heap: %s | Allocated-Heap: %s",
-		humanize.Bytes(memstats.Alloc),
-		humanize.Bytes(memstats.TotalAlloc),
-		humanize.Bytes(memstats.HeapAlloc),
-		humanize.Bytes(memstats.HeapSys))
-	platform := fmt.Sprintf("Host: %s | OS: %s | Arch: %s",
-		host,
-		runtime.GOOS,
-		runtime.GOARCH)
-	goruntime := fmt.Sprintf("Version: %s | CPUs: %s", runtime.Version(), strconv.Itoa(runtime.NumCPU()))
-	return map[string]string{
-		"PLATFORM": platform,
-		"RUNTIME":  goruntime,
-		"MEM":      mem,
-	}
+func enableLoggers() {
+	// Enable all loggers here.
+	enableConsoleLogger()
+	enableFileLogger()
+
+	// Add your logger here.
 }
 
 func findClosestCommands(command string) []string {
@@ -118,26 +109,18 @@ func findClosestCommands(command string) []string {
 }
 
 func registerApp() *cli.App {
-	// register all commands
+	// Register all commands.
 	registerCommand(serverCmd)
-	registerCommand(configCmd)
 	registerCommand(versionCmd)
 	registerCommand(updateCmd)
 
-	// register all flags
-	registerFlag(configFolderFlag)
-	registerFlag(addressFlag)
-	registerFlag(accessLogFlag)
-	registerFlag(certFlag)
-	registerFlag(keyFlag)
-
-	// set up app
+	// Set up app.
 	app := cli.NewApp()
 	app.Name = "Minio"
 	app.Author = "Minio.io"
-	app.Usage = "Cloud Storage Server for Micro Services."
-	app.Description = `Micro services environment provisions one Minio server per application instance. Scalability is achieved through large number of smaller personalized instances. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.`
-	app.Flags = flags
+	app.Usage = "Cloud Storage Server."
+	app.Description = `Minio is an Amazon S3 compatible object storage server. Use it to store photos, videos, VMs, containers, log files, or any blob of data as objects.`
+	app.Flags = append(minioFlags, globalFlags...)
 	app.Commands = commands
 	app.CustomAppHelpTemplate = minioHelpTemplate
 	app.CommandNotFound = func(ctx *cli.Context, command string) {
@@ -160,19 +143,20 @@ func checkMainSyntax(c *cli.Context) {
 		console.Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
 	}
 	if configPath == "" {
-		console.Fatalln("Config folder cannot be empty, please specify --config-folder <foldername>.")
+		console.Fatalln("Config folder cannot be empty, please specify --config-dir <foldername>.")
 	}
 }
 
-func main() {
-	probe.Init() // Set project's root source path.
-	probe.SetAppInfo("Release-Tag", minioReleaseTag)
-	probe.SetAppInfo("Commit-ID", minioShortCommitID)
+// mustGetProfilePath must get location that the profile will be written to.
+func mustGetProfilePath() string {
+	return filepath.Join(mustGetConfigPath(), globalMinioProfilePath)
+}
 
+func main() {
 	app := registerApp()
 	app.Before = func(c *cli.Context) error {
 		// Sets new config folder.
-		setGlobalConfigPath(c.GlobalString("config-folder"))
+		setGlobalConfigPath(c.GlobalString("config-dir"))
 
 		// Valid input arguments to main.
 		checkMainSyntax(c)
@@ -180,11 +164,44 @@ func main() {
 		// Migrate any old version of config / state files to newer format.
 		migrate()
 
+		// Initialize config.
+		err := initConfig()
+		fatalIf(err, "Unable to initialize minio config.")
+
+		// Enable all loggers by now.
+		enableLoggers()
+
+		// Initialize name space lock.
+		initNSLock()
+
+		// Set global quiet flag.
+		globalQuiet = c.Bool("quiet") || c.GlobalBool("quiet")
+
+		// Do not print update messages, if quiet flag is set.
+		if !globalQuiet {
+			// Do not print any errors in release update function.
+			noError := true
+			updateMsg := getReleaseUpdate(minioUpdateStableURL, noError)
+			if updateMsg.Update {
+				console.Println(updateMsg)
+			}
+		}
+
 		return nil
 	}
-	app.ExtraInfo = func() map[string]string {
-		return getSystemData()
+
+	// Set ``MINIO_PROFILE_DIR`` to the directory where profiling information should be persisted
+	profileDir := os.Getenv("MINIO_PROFILE_DIR")
+	// Enable profiler if ``MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
+	switch os.Getenv("MINIO_PROFILER") {
+	case "cpu":
+		defer profile.Start(profile.CPUProfile, profile.ProfilePath(profileDir)).Stop()
+	case "mem":
+		defer profile.Start(profile.MemProfile, profile.ProfilePath(profileDir)).Stop()
+	case "block":
+		defer profile.Start(profile.BlockProfile, profile.ProfilePath(profileDir)).Stop()
 	}
 
+	// Run the app - exit on error.
 	app.RunAndExitOnError()
 }

namespace-lock.go

@@ -0,0 +1,128 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"sync"
+)
+
+// nsParam - carries name space resource.
+type nsParam struct {
+	volume string
+	path   string
+}
+
+// nsLock - provides primitives for locking critical namespace regions.
+type nsLock struct {
+	*sync.RWMutex
+	ref uint
+}
+
+// nsLockMap - namespace lock map, provides primitives to Lock,
+// Unlock, RLock and RUnlock.
+type nsLockMap struct {
+	lockMap map[nsParam]*nsLock
+	mutex   *sync.Mutex
+}
+
+// Global name space lock.
+var nsMutex *nsLockMap
+
+// initNSLock - initialize name space lock map.
+func initNSLock() {
+	nsMutex = &nsLockMap{
+		lockMap: make(map[nsParam]*nsLock),
+		mutex:   &sync.Mutex{},
+	}
+}
+
+// Lock the namespace resource.
+func (n *nsLockMap) lock(volume, path string, readLock bool) {
+	n.mutex.Lock()
+
+	param := nsParam{volume, path}
+	nsLk, found := n.lockMap[param]
+	if !found {
+		nsLk = &nsLock{
+			RWMutex: &sync.RWMutex{},
+			ref:     0,
+		}
+		n.lockMap[param] = nsLk
+	}
+	nsLk.ref++ // Update ref count here to avoid multiple races.
+	// Unlock map before Locking NS which might block.
+	n.mutex.Unlock()
+
+	// Locking here can block.
+	if readLock {
+		nsLk.RLock()
+	} else {
+		nsLk.Lock()
+	}
+}
+
+// Unlock the namespace resource.
+func (n *nsLockMap) unlock(volume, path string, readLock bool) {
+	// nsLk.Unlock() will not block, hence locking the map for the entire function is fine.
+	n.mutex.Lock()
+	defer n.mutex.Unlock()
+
+	param := nsParam{volume, path}
+	if nsLk, found := n.lockMap[param]; found {
+		if readLock {
+			nsLk.RUnlock()
+		} else {
+			nsLk.Unlock()
+		}
+		if nsLk.ref == 0 {
+			errorIf(errors.New("Namespace reference count cannot be 0."), "Invalid reference count detected.")
+		}
+		if nsLk.ref != 0 {
+			nsLk.ref--
+		}
+		if nsLk.ref == 0 {
+			// Remove from the map if there are no more references.
+			delete(n.lockMap, param)
+		}
+	}
+}
+
+// Lock - locks the given resource for writes, using a previously
+// allocated name space lock or initializing a new one.
+func (n *nsLockMap) Lock(volume, path string) {
+	readLock := false
+	n.lock(volume, path, readLock)
+}
+
+// Unlock - unlocks any previously acquired write locks.
+func (n *nsLockMap) Unlock(volume, path string) {
+	readLock := false
+	n.unlock(volume, path, readLock)
+}
+
+// RLock - locks any previously acquired read locks.
+func (n *nsLockMap) RLock(volume, path string) {
+	readLock := true
+	n.lock(volume, path, readLock)
+}
+
+// RUnlock - unlocks any previously acquired read locks.
+func (n *nsLockMap) RUnlock(volume, path string) {
+	readLock := true
+	n.unlock(volume, path, readLock)
+}

notifier.go

@@ -18,69 +18,62 @@ package main
 
 import (
 	"fmt"
-	"math"
 	"runtime"
 	"strings"
 
 	"github.com/fatih/color"
-	"github.com/minio/minio/pkg/probe"
 	"github.com/olekukonko/ts"
 )
 
 // colorizeUpdateMessage - inspired from Yeoman project npm package https://github.com/yeoman/update-notifier
-func colorizeUpdateMessage(updateString string) (string, *probe.Error) {
-	// initialize coloring
+func colorizeUpdateMessage(updateString string) (string, error) {
+	// Initialize coloring.
 	cyan := color.New(color.FgCyan, color.Bold).SprintFunc()
 	yellow := color.New(color.FgYellow, color.Bold).SprintfFunc()
 
-	// calculate length without color coding, due to ANSI color characters padded to actual
-	// string the final length is wrong than the original string length
-	line1Str := fmt.Sprintf("  Update available: ")
-	line2Str := fmt.Sprintf("  Run \"%s\" to update. ", updateString)
+	// Calculate length without color coding, due to ANSI color
+	// characters padded to actual string the final length is wrong
+	// than the original string length.
+	line1Str := fmt.Sprintf("  New update: %s ", updateString)
 	line1Length := len(line1Str)
-	line2Length := len(line2Str)
 
-	// populate lines with color coding
-	line1InColor := line1Str
-	line2InColor := fmt.Sprintf("  Run \"%s\" to update. ", cyan(updateString))
+	// Populate lines with color coding.
+	line1InColor := fmt.Sprintf("  New update: %s ", cyan(updateString))
 
-	// calculate the rectangular box size
-	maxContentWidth := int(math.Max(float64(line1Length), float64(line2Length)))
+	// Calculate the rectangular box size.
+	maxContentWidth := line1Length
 	line1Rest := maxContentWidth - line1Length
-	line2Rest := maxContentWidth - line2Length
 
 	terminal, err := ts.GetSize()
 	if err != nil {
-		return "", probe.NewError(err)
+		return "", err
 	}
 
 	var message string
 	switch {
-	case len(line2Str) > terminal.Col():
-		message = "\n" + line1InColor + "\n" + line2InColor + "\n"
+	case len(line1Str) > terminal.Col():
+		message = "\n" + line1InColor + "\n"
 	default:
-		// on windows terminal turn off unicode characters
+		// On windows terminal turn off unicode characters.
 		var top, bottom, sideBar string
 		if runtime.GOOS == "windows" {
 			top = yellow("*" + strings.Repeat("*", maxContentWidth) + "*")
 			bottom = yellow("*" + strings.Repeat("*", maxContentWidth) + "*")
 			sideBar = yellow("|")
 		} else {
-			// color the rectangular box, use unicode characters here
+			// Color the rectangular box, use unicode characters here.
 			top = yellow("┏" + strings.Repeat("━", maxContentWidth) + "┓")
 			bottom = yellow("┗" + strings.Repeat("━", maxContentWidth) + "┛")
 			sideBar = yellow("┃")
 		}
-		// fill spaces to the rest of the area
+		// Fill spaces to the rest of the area.
 		spacePaddingLine1 := strings.Repeat(" ", line1Rest)
-		spacePaddingLine2 := strings.Repeat(" ", line2Rest)
 
-		// construct the final message
+		// Construct the final message.
 		message = "\n" + top + "\n" +
 			sideBar + line1InColor + spacePaddingLine1 + sideBar + "\n" +
-			sideBar + line2InColor + spacePaddingLine2 + sideBar + "\n" +
 			bottom + "\n"
 	}
-	// return the final message
+	// Return the final message.
 	return message, nil
 }

object-api-getobjectinfo_test.go

@@ -0,0 +1,156 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"testing"
+)
+
+// Wrapper for calling GetObjectInfo tests for both XL multiple disks and single node setup.
+func TestGetObjectInfo(t *testing.T) {
+	ExecObjectLayerTest(t, testGetObjectInfo)
+}
+
+// Testing GetObjectInfo().
+func testGetObjectInfo(obj ObjectLayer, instanceType string, t *testing.T) {
+	// This bucket is used for testing getObjectInfo operations.
+	err := obj.MakeBucket("test-getobjectinfo")
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+	_, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+	resultCases := []ObjectInfo{
+		// ObjectInfo -1.
+		// ObjectName set to a existing object in the test case (Test case 14).
+		{Bucket: "test-getobjectinfo", Name: "Asia/asiapics.jpg", ContentType: "image/jpeg", IsDir: false},
+	}
+	testCases := []struct {
+		bucketName string
+		objectName string
+
+		// Expected output of GetObjectInfo.
+		result ObjectInfo
+		err    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names ( Test number 1-4 ).
+		{".test", "", ObjectInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Test cases with valid but non-existing bucket names (Test number 5-7).
+		{"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false},
+		{"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false},
+		// Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9).
+		{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
+		{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
+		// Test cases with non-existing object name with existing bucket (Test number 10-12).
+		{"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false},
+		{"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false},
+		{"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false},
+		// Test case with existing bucket but object name set to a directory (Test number 13).
+		{"test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false},
+		// Valid case with existing object (Test number 14).
+		{"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true},
+	}
+	for i, testCase := range testCases {
+		result, err := obj.GetObjectInfo(testCase.bucketName, testCase.objectName)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if testCase.err.Error() != err.Error() {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error())
+			}
+		}
+
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.result.Bucket != result.Bucket {
+				t.Fatalf("Test %d: %s: Expected Bucket name to be '%s', but found '%s' instead", i+1, instanceType, testCase.result.Bucket, result.Bucket)
+			}
+			if testCase.result.Name != result.Name {
+				t.Errorf("Test %d: %s: Expected Object name to be %s, but instead found it to be %s", i+1, instanceType, testCase.result.Name, result.Name)
+			}
+			if testCase.result.ContentType != result.ContentType {
+				t.Errorf("Test %d: %s: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.ContentType, result.ContentType)
+			}
+			if testCase.result.IsDir != result.IsDir {
+				t.Errorf("Test %d: %s: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsDir, result.IsDir)
+			}
+		}
+	}
+}
+
+func BenchmarkGetObjectFS(b *testing.B) {
+	// Make a temporary directory to use as the obj.
+	directory, err := ioutil.TempDir("", "minio-benchmark-getobject")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the obj.
+	obj, err := newFSObjects(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Make a bucket and put in a few objects.
+	err = obj.MakeBucket("bucket")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	text := "Jack and Jill went up the hill / To fetch a pail of water."
+	hasher := md5.New()
+	hasher.Write([]byte(text))
+	metadata := make(map[string]string)
+	for i := 0; i < 10; i++ {
+		metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+		_, err = obj.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		var buffer = new(bytes.Buffer)
+		err = obj.GetObject("bucket", "object"+strconv.Itoa(i%10), 0, int64(len([]byte(text))), buffer)
+		if err != nil {
+			b.Error(err)
+		}
+		if buffer.Len() != len(text) {
+			b.Errorf("GetObject returned incorrect length %d (should be %d)\n", buffer.Len(), len(text))
+		}
+	}
+}

object-api-listobjects_test.go

@@ -0,0 +1,611 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Wrapper for calling ListObjects tests for both XL multiple disks and single node setup.
+func TestListObjects(t *testing.T) {
+	ExecObjectLayerTest(t, testListObjects)
+}
+
+// Unit test for ListObjects in general.
+func testListObjects(obj ObjectLayer, instanceType string, t *testing.T) {
+
+	// This bucket is used for testing ListObject operations.
+	err := obj.MakeBucket("test-bucket-list-object")
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+	// Will not store any objects in this bucket,
+	// Its to test ListObjects on an empty bucket.
+	err = obj.MakeBucket("empty-bucket")
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	tmpfile, err := ioutil.TempFile("", "simple-file.txt")
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	_, err = obj.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil)
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	_, err = obj.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	_, err = obj.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	for i := 0; i < 2; i++ {
+		key := "newPrefix" + strconv.Itoa(i)
+		_, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+	}
+	_, err = obj.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil)
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	for i := 0; i < 3; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+	}
+
+	// Formualting the result data set to be expected from ListObjects call inside the tests,
+	// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
+
+	resultCases := []ListObjectsInfo{
+		// ListObjectsResult-0.
+		// Testing for listing all objects in the bucket, (testCase 20,21,22).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-1.
+		// Used for asserting the truncated case, (testCase 23).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+			},
+		},
+		// ListObjectsResult-2.
+		// (TestCase 24).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+			},
+		},
+		// ListObjectsResult-3.
+		// (TestCase 25).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+		// ListObjectsResult-4.
+		// Again used for truncated case.
+		// (TestCase 26).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+			},
+		},
+		// ListObjectsResult-5.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix "new", (testCase 27-29).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-6.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix = "obj", (testCase 30).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-7.
+		// Used for Asserting prefixes and truncation.
+		// Used for test case with prefix = "new" and maxKeys = 1, (testCase 31).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+			},
+		},
+		// ListObjectsResult-8.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "obj0"},
+				{Name: "obj1"},
+			},
+		},
+		// ListObjectsResult-9.
+		// Used for asserting the case with marker, but without prefix.
+		//marker is set to "newPrefix0" in the testCase, (testCase 33).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-10.
+		//marker is set to "newPrefix1" in the testCase, (testCase 34).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-11.
+		//marker is set to "obj0" in the testCase, (testCase 35).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-12.
+		// Marker is set to "obj1" in the testCase, (testCase 36).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-13.
+		// Marker is set to "man" in the testCase, (testCase37).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-14.
+		// Marker is set to "Abc" in the testCase, (testCase 39).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-15.
+		// Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-16.
+		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-17.
+		// Used for asserting the case with marker, without prefix but with truncation.
+		// Marker =  "newPrefix0" & maxKeys = 3 in the testCase, (testCase42).
+		// Output truncated to 3 values.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+			},
+		},
+		// ListObjectsResult-18.
+		// Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43).
+		// Output truncated to 1 value.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-19.
+		// Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44).
+		// Output truncated to 1 value.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+			},
+		},
+		// ListObjectsResult-20.
+		// Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-21.
+		// Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-22.
+		// Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-23.
+		// Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+
+		// ListObjectsResult-24.
+		// Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+
+		// ListObjectsResult-25.
+		// Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+			},
+		},
+		// ListObjectsResult-26.
+		// prefix = "new" and delimiter is set in the testCase.(testCase 58).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+			},
+		},
+		// ListObjectsResult-27.
+		// Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/India-summer-photos-1"},
+			},
+		},
+		// ListObjectsResult-28.
+		// Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-29.
+		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimeter set, (testCase 61).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-30.
+		// Prefix and Delimiter is set to '/', (testCase 62).
+		{
+			IsTruncated: false,
+			Objects:     []ObjectInfo{},
+		},
+	}
+
+	testCases := []struct {
+		// Inputs to ListObjects.
+		bucketName string
+		prefix     string
+		marker     string
+		delimeter  string
+		maxKeys    int
+		// Expected output of ListObjects.
+		result ListObjectsInfo
+		err    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names ( Test number 1-4 ).
+		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Using an existing file for bucket name, but its not a directory (5).
+		{"simple-file.txt", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "simple-file.txt"}, false},
+		// Valid bucket names, but they donot exist (6-8).
+		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		// Valid, existing bucket, but sending invalid delimeter values (9-10).
+		// Empty string < "" > and forward slash < / > are the ony two valid arguments for delimeter.
+		{"test-bucket-list-object", "", "", "*", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "*"), false},
+		{"test-bucket-list-object", "", "", "-", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "-"), false},
+		// Testing for failure cases with both perfix and marker (13).
+		// The prefix and marker combination to be valid it should satisy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
+		// Setting a non-existing directory to be prefix (14-15).
+		{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
+		{"empty-bucket", "europe/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
+		// Testing on empty bucket, that is, bucket without any objects in it (16).
+		{"empty-bucket", "", "", "", 0, ListObjectsInfo{}, nil, true},
+		// Setting maxKeys to negative value (17-18).
+		{"empty-bucket", "", "", "", -1, ListObjectsInfo{}, nil, true},
+		{"empty-bucket", "", "", "", 1, ListObjectsInfo{}, nil, true},
+		// Setting maxKeys to a very large value (19).
+		{"empty-bucket", "", "", "", 1111000000000000, ListObjectsInfo{}, nil, true},
+		// Testing for all 7 objects in the bucket (20).
+		{"test-bucket-list-object", "", "", "", 9, resultCases[0], nil, true},
+		//Testing for negative value of maxKey, this should set maxKeys to listObjectsLimit (21).
+		{"test-bucket-list-object", "", "", "", -1, resultCases[0], nil, true},
+		// Testing for very large value of maxKey, this should set maxKeys to listObjectsLimit (22).
+		{"test-bucket-list-object", "", "", "", 1234567891011, resultCases[0], nil, true},
+		// Testing for trancated value (23-26).
+		{"test-bucket-list-object", "", "", "", 5, resultCases[1], nil, true},
+		{"test-bucket-list-object", "", "", "", 4, resultCases[2], nil, true},
+		{"test-bucket-list-object", "", "", "", 3, resultCases[3], nil, true},
+		{"test-bucket-list-object", "", "", "", 1, resultCases[4], nil, true},
+		// Testing with prefix (27-30).
+		{"test-bucket-list-object", "new", "", "", 3, resultCases[5], nil, true},
+		{"test-bucket-list-object", "new", "", "", 4, resultCases[5], nil, true},
+		{"test-bucket-list-object", "new", "", "", 5, resultCases[5], nil, true},
+		{"test-bucket-list-object", "obj", "", "", 3, resultCases[6], nil, true},
+		// Testing with prefix and truncation (31-32).
+		{"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true},
+		{"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true},
+		// Testing with marker, but without prefix and truncation (33-37).
+		{"test-bucket-list-object", "", "newPrefix0", "", 5, resultCases[9], nil, true},
+		{"test-bucket-list-object", "", "newPrefix1", "", 4, resultCases[10], nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 2, resultCases[11], nil, true},
+		{"test-bucket-list-object", "", "obj1", "", 1, resultCases[12], nil, true},
+		{"test-bucket-list-object", "", "man", "", 10, resultCases[13], nil, true},
+		// Marker being set to a value which is greater than and all object names when sorted (38).
+		// Expected to send an empty response in this case.
+		{"test-bucket-list-object", "", "zen", "", 10, ListObjectsInfo{}, nil, true},
+		// Marker being set to a value which is lesser than and all object names when sorted (39).
+		// Expected to send all the objects in the bucket in this case.
+		{"test-bucket-list-object", "", "Abc", "", 10, resultCases[14], nil, true},
+		// Marker is to a hierarhical value (40-41).
+		{"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "", 10, resultCases[15], nil, true},
+		{"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", 10, resultCases[16], nil, true},
+		// Testing with marker and truncation, but no prefix (42-44).
+		{"test-bucket-list-object", "", "newPrefix0", "", 3, resultCases[17], nil, true},
+		{"test-bucket-list-object", "", "newPrefix1", "", 1, resultCases[18], nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 1, resultCases[19], nil, true},
+		// Testing with both marker and prefix, but without truncation (45-47).
+		// The valid combination of marker and prefix should satisfy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "obj", "obj0", "", 2, resultCases[20], nil, true},
+		{"test-bucket-list-object", "obj", "obj1", "", 1, resultCases[21], nil, true},
+		{"test-bucket-list-object", "new", "newPrefix0", "", 2, resultCases[22], nil, true},
+		// Testing with maxKeys set to 0 (48-54).
+		// The parameters have to valid.
+		{"test-bucket-list-object", "", "obj1", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "new", "", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "obj0", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "obj1", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "new", "newPrefix0", "", 0, ListObjectsInfo{}, nil, true},
+		// Tests on hierarchical key names as prefix.
+		// Without delimteter the code should recurse into the prefix Dir.
+		// Tests with prefix, but without delimiter (55-56).
+		{"test-bucket-list-object", "Asia/India/", "", "", 10, resultCases[23], nil, true},
+		{"test-bucket-list-object", "Asia", "", "", 10, resultCases[24], nil, true},
+		// Tests with prefix and delimiter (57-59).
+		// With delimeter the code shouldnot recurse into the sub-directories of prefix Dir.
+		{"test-bucket-list-object", "Asia", "", "/", 10, resultCases[25], nil, true},
+		{"test-bucket-list-object", "new", "", "/", 10, resultCases[26], nil, true},
+		{"test-bucket-list-object", "Asia/India/", "", "/", 10, resultCases[27], nil, true},
+		// Test with marker set as hierarhical value and with delimiter. (60-61)
+		{"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "/", 10, resultCases[28], nil, true},
+		{"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "/", 10, resultCases[29], nil, true},
+		// Test with prefix and delimiter set to '/'. (62)
+		{"test-bucket-list-object", "/", "", "/", 10, resultCases[30], nil, true},
+	}
+
+	for i, testCase := range testCases {
+		result, err := obj.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if !strings.Contains(err.Error(), testCase.err.Error()) {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error())
+			}
+		}
+		// Since there are cases for which ListObjects fails, this is
+		// necessary. Test passes as expected, but the output values
+		// are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			// The length of the expected ListObjectsResult.Objects
+			// should match in both expected result from test cases
+			// and in the output. On failure calling t.Fatalf,
+			// otherwise it may lead to index out of range error in
+			// assertion following this.
+			if len(testCase.result.Objects) != len(result.Objects) {
+				t.Fatalf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.result.Objects), len(result.Objects))
+			}
+			for j := 0; j < len(testCase.result.Objects); j++ {
+				if testCase.result.Objects[j].Name != result.Objects[j].Name {
+					t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name)
+				}
+			}
+			if testCase.result.IsTruncated != result.IsTruncated {
+				t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsTruncated, result.IsTruncated)
+			}
+
+		}
+	}
+
+}
+
+func BenchmarkListObjects(b *testing.B) {
+	// Make a temporary directory to use as the obj.
+	directory, err := ioutil.TempDir("", "minio-list-benchmark")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the obj.
+	obj, err := newFSObjects(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Create a bucket.
+	err = obj.MakeBucket("ls-benchmark-bucket")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	for i := 0; i < 20000; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = obj.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+
+	// List the buckets over and over and over.
+	for i := 0; i < b.N; i++ {
+		_, err = obj.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

object-api_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,31 +14,54 @@
  * limitations under the License.
  */
 
-package fs
+package main
 
 import (
 	"io/ioutil"
 	"os"
-	"testing"
 
 	. "gopkg.in/check.v1"
 )
 
-func Test(t *testing.T) { TestingT(t) }
-
 type MySuite struct{}
 
 var _ = Suite(&MySuite{})
 
-func (s *MySuite) TestAPISuite(c *C) {
+func (s *MySuite) TestFSAPISuite(c *C) {
 	var storageList []string
-	create := func() Filesystem {
-		path, e := ioutil.TempDir(os.TempDir(), "minio-")
-		c.Check(e, IsNil)
-		storageList = append(storageList, path)
-		store, err := New(path, 0)
+
+	// Initialize name space lock.
+	initNSLock()
+
+	create := func() ObjectLayer {
+		path, err := ioutil.TempDir(os.TempDir(), "minio-")
 		c.Check(err, IsNil)
-		return store
+		objAPI, err := newFSObjects(path)
+		c.Check(err, IsNil)
+		storageList = append(storageList, path)
+		return objAPI
+	}
+	APITestSuite(c, create)
+	defer removeRoots(c, storageList)
+}
+
+func (s *MySuite) TestXLAPISuite(c *C) {
+	var storageList []string
+
+	// Initialize name space lock.
+	initNSLock()
+
+	create := func() ObjectLayer {
+		var nDisks = 16 // Maximum disks.
+		var erasureDisks []string
+		for i := 0; i < nDisks; i++ {
+			path, err := ioutil.TempDir(os.TempDir(), "minio-")
+			c.Check(err, IsNil)
+			erasureDisks = append(erasureDisks, path)
+		}
+		objAPI, err := newXLObjects(erasureDisks)
+		c.Check(err, IsNil)
+		return objAPI
 	}
 	APITestSuite(c, create)
 	defer removeRoots(c, storageList)
@@ -46,7 +69,6 @@ func (s *MySuite) TestAPISuite(c *C) {
 
 func removeRoots(c *C, roots []string) {
 	for _, root := range roots {
-		err := os.RemoveAll(root)
-		c.Check(err, IsNil)
+		os.RemoveAll(root)
 	}
 }

object-common.go

@@ -0,0 +1,137 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+const (
+	// Block size used for all internal operations version 1.
+	blockSizeV1 = 10 * 1024 * 1024 // 10MiB.
+)
+
+// House keeping code needed for FS.
+func fsHouseKeeping(storageDisk StorageAPI) error {
+	// Attempt to create `.minio`.
+	err := storageDisk.MakeVol(minioMetaBucket)
+	if err != nil {
+		if err != errVolumeExists && err != errDiskNotFound {
+			return err
+		}
+	}
+	// Cleanup all temp entries upon start.
+	err = cleanupDir(storageDisk, minioMetaBucket, tmpMetaPrefix)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Depending on the disk type network or local, initialize storage API.
+func newStorageAPI(disk string) (storage StorageAPI, err error) {
+	if !strings.ContainsRune(disk, ':') || filepath.VolumeName(disk) != "" {
+		// Initialize filesystem storage API.
+		return newPosix(disk)
+	}
+	// Initialize rpc client storage API.
+	return newRPCClient(disk)
+}
+
+// House keeping code needed for XL.
+func xlHouseKeeping(storageDisks []StorageAPI) error {
+	// This happens for the first time, but keep this here since this
+	// is the only place where it can be made expensive optimizing all
+	// other calls. Create minio meta volume, if it doesn't exist yet.
+	var wg = &sync.WaitGroup{}
+
+	// Initialize errs to collect errors inside go-routine.
+	var errs = make([]error, len(storageDisks))
+
+	// Initialize all disks in parallel.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			errs[index] = errDiskNotFound
+			continue
+		}
+		wg.Add(1)
+		go func(index int, disk StorageAPI) {
+			// Indicate this wait group is done.
+			defer wg.Done()
+
+			// Attempt to create `.minio`.
+			err := disk.MakeVol(minioMetaBucket)
+			if err != nil && err != errVolumeExists && err != errDiskNotFound {
+				errs[index] = err
+				return
+			}
+			// Cleanup all temp entries upon start.
+			err = cleanupDir(disk, minioMetaBucket, tmpMetaPrefix)
+			if err != nil {
+				errs[index] = err
+				return
+			}
+			errs[index] = nil
+		}(index, disk)
+	}
+
+	// Wait for all cleanup to finish.
+	wg.Wait()
+
+	// Return upon first error.
+	for _, err := range errs {
+		if err == nil {
+			continue
+		}
+		return toObjectErr(err, minioMetaBucket, tmpMetaPrefix)
+	}
+
+	// Return success here.
+	return nil
+}
+
+// Cleanup a directory recursively.
+func cleanupDir(storage StorageAPI, volume, dirPath string) error {
+	var delFunc func(string) error
+	// Function to delete entries recursively.
+	delFunc = func(entryPath string) error {
+		if !strings.HasSuffix(entryPath, slashSeparator) {
+			// No trailing "/" means that this is a file which can be deleted.
+			return storage.DeleteFile(volume, entryPath)
+		}
+		// If it's a directory, list and call delFunc() for each entry.
+		entries, err := storage.ListDir(volume, entryPath)
+		if err != nil {
+			if err == errFileNotFound {
+				// if dirPath prefix never existed.
+				return nil
+			}
+			return err
+		}
+		for _, entry := range entries {
+			err = delFunc(pathJoin(entryPath, entry))
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	err := delFunc(retainSlash(pathJoin(dirPath)))
+	return err
+}

object-datatypes.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,41 +14,36 @@
  * limitations under the License.
  */
 
-package fs
+package main
 
-import (
-	"os"
-	"time"
-)
+import "time"
 
-// BucketMetadata - name and create date
-type BucketMetadata struct {
+// StorageInfo - represents total capacity of underlying storage.
+type StorageInfo struct {
+	Total int64 // Total disk space.
+	Free  int64 // Free total available disk space.
+}
+
+// BucketInfo - bucket name and create date
+type BucketInfo struct {
 	Name    string
 	Created time.Time
 }
 
-// ObjectMetadata - object key and its relevant metadata
-type ObjectMetadata struct {
-	Bucket string
-	Object string
-
-	ContentType  string
-	LastModified time.Time
-	Mode         os.FileMode
-	MD5          string
-	Size         int64
+// ObjectInfo - object info.
+type ObjectInfo struct {
+	Bucket          string
+	Name            string
+	ModTime         time.Time
+	Size            int64
+	IsDir           bool
+	MD5Sum          string
+	ContentType     string
+	ContentEncoding string
 }
 
-// PartMetadata - various types of individual part resources
-type PartMetadata struct {
-	PartNumber   int
-	LastModified time.Time
-	ETag         string
-	Size         int64
-}
-
-// ObjectResourcesMetadata - various types of object resources
-type ObjectResourcesMetadata struct {
+// ListPartsInfo - various types of object resources.
+type ListPartsInfo struct {
 	Bucket               string
 	Object               string
 	UploadID             string
@@ -58,20 +53,12 @@ type ObjectResourcesMetadata struct {
 	MaxParts             int
 	IsTruncated          bool
 
-	Part         []PartMetadata
+	Parts        []partInfo
 	EncodingType string
 }
 
-// UploadMetadata container capturing metadata on in progress multipart upload in a given bucket
-type UploadMetadata struct {
-	Object       string
-	UploadID     string
-	StorageClass string
-	Initiated    time.Time
-}
-
-// BucketMultipartResourcesMetadata - various types of bucket resources for inprogress multipart uploads
-type BucketMultipartResourcesMetadata struct {
+// ListMultipartsInfo - various types of bucket resources for inprogress multipart uploads.
+type ListMultipartsInfo struct {
 	KeyMarker          string
 	UploadIDMarker     string
 	NextKeyMarker      string
@@ -79,34 +66,50 @@ type BucketMultipartResourcesMetadata struct {
 	EncodingType       string
 	MaxUploads         int
 	IsTruncated        bool
-	Upload             []*UploadMetadata
+	Uploads            []uploadMetadata
 	Prefix             string
 	Delimiter          string
 	CommonPrefixes     []string
 }
 
-// ListObjectsResult - container for list object request results.
-type ListObjectsResult struct {
+// ListObjectsInfo - container for list objects.
+type ListObjectsInfo struct {
 	IsTruncated bool
 	NextMarker  string
-	Objects     []ObjectMetadata
+	Objects     []ObjectInfo
 	Prefixes    []string
 }
 
-// CompletePart - completed part container
-type CompletePart struct {
+// partInfo - various types of individual part resources.
+type partInfo struct {
+	PartNumber   int
+	LastModified time.Time
+	ETag         string
+	Size         int64
+}
+
+// uploadMetadata container capturing metadata on in progress multipart upload in a given bucket
+type uploadMetadata struct {
+	Object       string
+	UploadID     string
+	StorageClass string
+	Initiated    time.Time
+}
+
+// completePart - completed part container.
+type completePart struct {
 	PartNumber int
 	ETag       string
 }
 
 // completedParts is a sortable interface for Part slice
-type completedParts []CompletePart
+type completedParts []completePart
 
 func (a completedParts) Len() int           { return len(a) }
 func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
 
-// CompleteMultipartUpload container for completing multipart upload
-type CompleteMultipartUpload struct {
-	Part []CompletePart
+// completeMultipartUpload container for completing multipart upload
+type completeMultipartUpload struct {
+	Parts []completePart `xml:"Part"`
 }

object-errors.go

@@ -0,0 +1,258 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"io"
+)
+
+// Converts underlying storage error. Convenience function written to
+// handle all cases where we have known types of errors returned by
+// underlying storage layer.
+func toObjectErr(err error, params ...string) error {
+	switch err {
+	case errVolumeNotFound:
+		if len(params) >= 1 {
+			return BucketNotFound{Bucket: params[0]}
+		}
+	case errVolumeNotEmpty:
+		if len(params) >= 1 {
+			return BucketNotEmpty{Bucket: params[0]}
+		}
+	case errVolumeExists:
+		if len(params) >= 1 {
+			return BucketExists{Bucket: params[0]}
+		}
+	case errDiskFull:
+		return StorageFull{}
+	case errIsNotRegular, errFileAccessDenied:
+		if len(params) >= 2 {
+			return ObjectExistsAsDirectory{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		}
+	case errFileNotFound:
+		if len(params) >= 2 {
+			return ObjectNotFound{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		}
+	case errFileNameTooLong:
+		if len(params) >= 2 {
+			return ObjectNameInvalid{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		}
+	case errXLReadQuorum:
+		return InsufficientReadQuorum{}
+	case errXLWriteQuorum:
+		return InsufficientWriteQuorum{}
+	case io.ErrUnexpectedEOF, io.ErrShortWrite:
+		return IncompleteBody{}
+	}
+	return err
+}
+
+// StorageFull storage ran out of space.
+type StorageFull struct{}
+
+func (e StorageFull) Error() string {
+	return "Storage reached its minimum free disk threshold."
+}
+
+// InsufficientReadQuorum storage cannot satisfy quorum for read operation.
+type InsufficientReadQuorum struct{}
+
+func (e InsufficientReadQuorum) Error() string {
+	return "Storage resources are insufficient for the read operation."
+}
+
+// InsufficientWriteQuorum storage cannot satisfy quorum for write operation.
+type InsufficientWriteQuorum struct{}
+
+func (e InsufficientWriteQuorum) Error() string {
+	return "Storage resources are insufficient for the write operation."
+}
+
+// GenericError - generic object layer error.
+type GenericError struct {
+	Bucket string
+	Object string
+}
+
+// BucketNotFound bucket does not exist.
+type BucketNotFound GenericError
+
+func (e BucketNotFound) Error() string {
+	return "Bucket not found: " + e.Bucket
+}
+
+// BucketNotEmpty bucket is not empty.
+type BucketNotEmpty GenericError
+
+func (e BucketNotEmpty) Error() string {
+	return "Bucket not empty: " + e.Bucket
+}
+
+// ObjectNotFound object does not exist.
+type ObjectNotFound GenericError
+
+func (e ObjectNotFound) Error() string {
+	return "Object not found: " + e.Bucket + "#" + e.Object
+}
+
+// ObjectExistsAsDirectory object already exists as a directory.
+type ObjectExistsAsDirectory GenericError
+
+func (e ObjectExistsAsDirectory) Error() string {
+	return "Object exists on : " + e.Bucket + " as directory " + e.Object
+}
+
+// BucketExists bucket exists.
+type BucketExists GenericError
+
+func (e BucketExists) Error() string {
+	return "Bucket exists: " + e.Bucket
+}
+
+// BadDigest - Content-MD5 you specified did not match what we received.
+type BadDigest struct {
+	ExpectedMD5   string
+	CalculatedMD5 string
+}
+
+func (e BadDigest) Error() string {
+	return "Bad digest: Expected " + e.ExpectedMD5 + " is not valid with what we calculated " + e.CalculatedMD5
+}
+
+// UnsupportedDelimiter - unsupported delimiter.
+type UnsupportedDelimiter struct {
+	Delimiter string
+}
+
+func (e UnsupportedDelimiter) Error() string {
+	return fmt.Sprintf("delimiter '%s' is not supported. Only '/' is supported", e.Delimiter)
+}
+
+// InvalidUploadIDKeyCombination - invalid upload id and key marker combination.
+type InvalidUploadIDKeyCombination struct {
+	UploadIDMarker, KeyMarker string
+}
+
+func (e InvalidUploadIDKeyCombination) Error() string {
+	return fmt.Sprintf("Invalid combination of uploadID marker '%s' and marker '%s'", e.UploadIDMarker, e.KeyMarker)
+}
+
+// InvalidMarkerPrefixCombination - invalid marker and prefix combination.
+type InvalidMarkerPrefixCombination struct {
+	Marker, Prefix string
+}
+
+func (e InvalidMarkerPrefixCombination) Error() string {
+	return fmt.Sprintf("Invalid combination of marker '%s' and prefix '%s'", e.Marker, e.Prefix)
+}
+
+// BucketPolicyNotFound - no bucket policy found.
+type BucketPolicyNotFound GenericError
+
+func (e BucketPolicyNotFound) Error() string {
+	return "No bucket policy found for bucket: " + e.Bucket
+}
+
+/// Bucket related errors.
+
+// BucketNameInvalid - bucketname provided is invalid.
+type BucketNameInvalid GenericError
+
+// Return string an error formatted as the given text.
+func (e BucketNameInvalid) Error() string {
+	return "Bucket name invalid: " + e.Bucket
+}
+
+/// Object related errors.
+
+// ObjectNameInvalid - object name provided is invalid.
+type ObjectNameInvalid GenericError
+
+// Return string an error formatted as the given text.
+func (e ObjectNameInvalid) Error() string {
+	return "Object name invalid: " + e.Bucket + "#" + e.Object
+}
+
+// UnExpectedDataSize - Reader contains more/less data than specified.
+type UnExpectedDataSize struct {
+	Size int
+}
+
+func (e UnExpectedDataSize) Error() string {
+	return fmt.Sprintf("Contains more data than specified size of %d bytes.", e.Size)
+}
+
+// IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header.
+type IncompleteBody GenericError
+
+// Return string an error formatted as the given text.
+func (e IncompleteBody) Error() string {
+	return e.Bucket + "#" + e.Object + "has incomplete body"
+}
+
+/// Multipart related errors.
+
+// MalformedUploadID malformed upload id.
+type MalformedUploadID struct {
+	UploadID string
+}
+
+func (e MalformedUploadID) Error() string {
+	return "Malformed upload id " + e.UploadID
+}
+
+// InvalidUploadID invalid upload id.
+type InvalidUploadID struct {
+	UploadID string
+}
+
+func (e InvalidUploadID) Error() string {
+	return "Invalid upload id " + e.UploadID
+}
+
+// InvalidPart One or more of the specified parts could not be found
+type InvalidPart struct{}
+
+func (e InvalidPart) Error() string {
+	return "One or more of the specified parts could not be found"
+}
+
+// InvalidPartOrder parts are not ordered as Requested
+type InvalidPartOrder struct {
+	UploadID string
+}
+
+func (e InvalidPartOrder) Error() string {
+	return "Invalid part order sent for " + e.UploadID
+}
+
+// PartTooSmall - error if part size is less than 5MB.
+type PartTooSmall struct{}
+
+func (e PartTooSmall) Error() string {
+	return "Part size should be atleast 5MB"
+}

object-interface.go

@@ -0,0 +1,46 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "io"
+
+// ObjectLayer implements primitives for object API layer.
+type ObjectLayer interface {
+	// Storage operations.
+	StorageInfo() StorageInfo
+
+	// Bucket operations.
+	MakeBucket(bucket string) error
+	GetBucketInfo(bucket string) (bucketInfo BucketInfo, err error)
+	ListBuckets() (buckets []BucketInfo, err error)
+	DeleteBucket(bucket string) error
+	ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
+
+	// Object operations.
+	GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
+	GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
+	PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string) (md5 string, err error)
+	DeleteObject(bucket, object string) error
+
+	// Multipart operations.
+	ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)
+	NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error)
+	PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (md5 string, err error)
+	ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error)
+	AbortMultipartUpload(bucket, object, uploadID string) error
+	CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (md5 string, err error)
+}

object-utils.go

@@ -0,0 +1,159 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"path"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/skyrings/skyring-common/tools/uuid"
+)
+
+const (
+	// Minio meta bucket.
+	minioMetaBucket = ".minio"
+	// Multipart meta prefix.
+	mpartMetaPrefix = "multipart"
+	// Tmp meta prefix.
+	tmpMetaPrefix = "tmp"
+)
+
+// validBucket regexp.
+var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
+
+// IsValidBucketName verifies a bucket name in accordance with Amazon's
+// requirements. It must be 3-63 characters long, can contain dashes
+// and periods, but must begin and end with a lowercase letter or a number.
+// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
+func IsValidBucketName(bucket string) bool {
+	if len(bucket) < 3 || len(bucket) > 63 {
+		return false
+	}
+	if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
+		return false
+	}
+	return validBucket.MatchString(bucket)
+}
+
+// IsValidObjectName verifies an object name in accordance with Amazon's
+// requirements. It cannot exceed 1024 characters and must be a valid UTF8
+// string.
+//
+// See:
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
+//
+// You should avoid the following characters in a key name because of
+// significant special handling for consistency across all
+// applications.
+//
+// Rejects strings with following characters.
+//
+// - Backslash ("\")
+// - Caret ("^")
+// - Grave accent / back tick ("`")
+// - Vertical bar / pipe ("|")
+//
+// Minio does not support object names with trailing "/".
+func IsValidObjectName(object string) bool {
+	if len(object) == 0 {
+		return false
+	}
+	if strings.HasSuffix(object, slashSeparator) {
+		return false
+	}
+	if strings.HasPrefix(object, slashSeparator) {
+		return false
+	}
+	return IsValidObjectPrefix(object)
+}
+
+// IsValidObjectPrefix verifies whether the prefix is a valid object name.
+// Its valid to have a empty prefix.
+func IsValidObjectPrefix(object string) bool {
+	if len(object) > 1024 {
+		return false
+	}
+	if !utf8.ValidString(object) {
+		return false
+	}
+	// Reject unsupported characters in object name.
+	if strings.ContainsAny(object, "`^*|\\\"") {
+		return false
+	}
+	return true
+}
+
+// Slash separator.
+const slashSeparator = "/"
+
+// retainSlash - retains slash from a path.
+func retainSlash(s string) string {
+	return strings.TrimSuffix(s, slashSeparator) + slashSeparator
+}
+
+// pathJoin - like path.Join() but retains trailing "/" of the last element
+func pathJoin(elem ...string) string {
+	trailingSlash := ""
+	if len(elem) > 0 {
+		if strings.HasSuffix(elem[len(elem)-1], slashSeparator) {
+			trailingSlash = "/"
+		}
+	}
+	return path.Join(elem...) + trailingSlash
+}
+
+// getUUID() - get a unique uuid.
+func getUUID() (uuidStr string) {
+	for {
+		uuid, err := uuid.New()
+		if err != nil {
+			errorIf(err, "Unable to initialize uuid")
+			continue
+		}
+		uuidStr = uuid.String()
+		break
+	}
+	return uuidStr
+}
+
+// Create an s3 compatible MD5sum for complete multipart transaction.
+func completeMultipartMD5(parts ...completePart) (string, error) {
+	var finalMD5Bytes []byte
+	for _, part := range parts {
+		md5Bytes, err := hex.DecodeString(part.ETag)
+		if err != nil {
+			return "", err
+		}
+		finalMD5Bytes = append(finalMD5Bytes, md5Bytes...)
+	}
+	md5Hasher := md5.New()
+	md5Hasher.Write(finalMD5Bytes)
+	s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(parts))
+	return s3MD5, nil
+}
+
+// byBucketName is a collection satisfying sort.Interface.
+type byBucketName []BucketInfo
+
+func (d byBucketName) Len() int           { return len(d) }
+func (d byBucketName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d byBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }

object-utils_test.go

@@ -0,0 +1,107 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"testing"
+)
+
+// Tests validate bucket name.
+func TestIsValidBucketName(t *testing.T) {
+	testCases := []struct {
+		bucketName string
+		shouldPass bool
+	}{
+		// cases which should pass the test.
+		// passing in valid bucket names.
+		{"lol", true},
+		{"1-this-is-valid", true},
+		{"1-this-too-is-valid-1", true},
+		{"this.works.too.1", true},
+		{"1234567", true},
+		{"123", true},
+		{"s3-eu-west-1.amazonaws.com", true},
+		{"ideas-are-more-powerful-than-guns", true},
+		{"testbucket", true},
+		{"1bucket", true},
+		{"bucket1", true},
+		// cases for which test should fail.
+		// passing invalid bucket names.
+		{"------", false},
+		{"$this-is-not-valid-too", false},
+		{"contains-$-dollar", false},
+		{"contains-^-carrot", false},
+		{"contains-$-dollar", false},
+		{"contains-$-dollar", false},
+		{"......", false},
+		{"", false},
+		{"a", false},
+		{"ab", false},
+		{".starts-with-a-dot", false},
+		{"ends-with-a-dot.", false},
+		{"ends-with-a-dash-", false},
+		{"-starts-with-a-dash", false},
+		{"THIS-BEINGS-WITH-UPPERCASe", false},
+		{"tHIS-ENDS-WITH-UPPERCASE", false},
+		{"ThisBeginsAndEndsWithUpperCase", false},
+		{"una ñina", false},
+		{"lalalallalallalalalallalallalala-theString-size-is-greater-than-64", false},
+	}
+
+	for i, testCase := range testCases {
+		isValidBucketName := IsValidBucketName(testCase.bucketName)
+		if testCase.shouldPass && !isValidBucketName {
+			t.Errorf("Test case %d: Expected \"%s\" to be a valid bucket name", i+1, testCase.bucketName)
+		}
+		if !testCase.shouldPass && isValidBucketName {
+			t.Errorf("Test case %d: Expected bucket name \"%s\" to be invalid", i+1, testCase.bucketName)
+		}
+	}
+}
+
+// Tests for validate object name.
+func TestIsValidObjectName(t *testing.T) {
+	testCases := []struct {
+		objectName string
+		shouldPass bool
+	}{
+		// cases which should pass the test.
+		// passing in valid object name.
+		{"object", true},
+		{"The Shining Script <v1>.pdf", true},
+		{"Cost Benefit Analysis (2009-2010).pptx", true},
+		{"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", true},
+		{"SHØRT", true},
+		{"There are far too many object names, and far too few bucket names!", true},
+		// cases for which test should fail.
+		// passing invalid object names.
+		{"", false},
+		{"a/b/c/", false},
+		{"/a/b/c", false},
+		{string([]byte{0xff, 0xfe, 0xfd}), false},
+	}
+
+	for i, testCase := range testCases {
+		isValidObjectName := IsValidObjectName(testCase.objectName)
+		if testCase.shouldPass && !isValidObjectName {
+			t.Errorf("Test case %d: Expected \"%s\" to be a valid object name", i+1, testCase.objectName)
+		}
+		if !testCase.shouldPass && isValidObjectName {
+			t.Errorf("Test case %d: Expected object name \"%s\" to be invalid", i+1, testCase.objectName)
+		}
+	}
+}

object_api_suite_test.go

@@ -0,0 +1,432 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"math/rand"
+	"strconv"
+
+	"gopkg.in/check.v1"
+)
+
+// APITestSuite - collection of API tests.
+func APITestSuite(c *check.C, create func() ObjectLayer) {
+	testMakeBucket(c, create)
+	testMultipleObjectCreation(c, create)
+	testPaging(c, create)
+	testObjectOverwriteWorks(c, create)
+	testNonExistantBucketOperations(c, create)
+	testBucketRecreateFails(c, create)
+	testPutObjectInSubdir(c, create)
+	testListBuckets(c, create)
+	testListBucketsOrder(c, create)
+	testListObjectsTestsForNonExistantBucket(c, create)
+	testNonExistantObjectInBucket(c, create)
+	testGetDirectoryReturnsObjectNotFound(c, create)
+	testDefaultContentType(c, create)
+	testMultipartObjectCreation(c, create)
+	testMultipartObjectAbort(c, create)
+}
+
+// Tests validate bucket creation.
+func testMakeBucket(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	err := obj.MakeBucket("bucket-unknown")
+	c.Assert(err, check.IsNil)
+
+}
+
+// Tests validate creation of part files during Multipart operation.
+func testMultipartObjectCreation(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	err := obj.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+	uploadID, err := obj.NewMultipartUpload("bucket", "key", nil)
+	c.Assert(err, check.IsNil)
+	// Create a byte array of 5MB.
+	data := bytes.Repeat([]byte("0123456789abcdef"), 5*1024*1024/16)
+	completedParts := completeMultipartUpload{}
+	for i := 1; i <= 10; i++ {
+		hasher := md5.New()
+		hasher.Write(data)
+		expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		var calculatedMD5sum string
+		calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(data)), bytes.NewBuffer(data), expectedMD5Sumhex)
+		c.Assert(err, check.IsNil)
+		c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
+		completedParts.Parts = append(completedParts.Parts, completePart{PartNumber: i, ETag: calculatedMD5sum})
+	}
+	md5Sum, err := obj.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts)
+	c.Assert(err, check.IsNil)
+	c.Assert(md5Sum, check.Equals, "7d364cb728ce42a74a96d22949beefb2-10")
+}
+
+// Tests validate abortion of Multipart operation.
+func testMultipartObjectAbort(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	err := obj.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+	uploadID, err := obj.NewMultipartUpload("bucket", "key", nil)
+	c.Assert(err, check.IsNil)
+
+	parts := make(map[int]string)
+	metadata := make(map[string]string)
+	for i := 1; i <= 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		hasher.Write([]byte(randomString))
+		expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		metadata["md5"] = expectedMD5Sumhex
+		var calculatedMD5sum string
+		calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
+		c.Assert(err, check.IsNil)
+		c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
+		parts[i] = expectedMD5Sumhex
+	}
+	err = obj.AbortMultipartUpload("bucket", "key", uploadID)
+	c.Assert(err, check.IsNil)
+}
+
+// Tests validate object creation.
+func testMultipleObjectCreation(c *check.C, create func() ObjectLayer) {
+	objects := make(map[string][]byte)
+	obj := create()
+	err := obj.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+	for i := 0; i < 10; i++ {
+		randomPerm := rand.Perm(100)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		hasher.Write([]byte(randomString))
+		expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		key := "obj" + strconv.Itoa(i)
+		objects[key] = []byte(randomString)
+		metadata := make(map[string]string)
+		metadata["md5Sum"] = expectedMD5Sumhex
+		var md5Sum string
+		md5Sum, err = obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata)
+		c.Assert(err, check.IsNil)
+		c.Assert(md5Sum, check.Equals, expectedMD5Sumhex)
+	}
+
+	for key, value := range objects {
+		var byteBuffer bytes.Buffer
+		err = obj.GetObject("bucket", key, 0, int64(len(value)), &byteBuffer)
+		c.Assert(err, check.IsNil)
+		c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
+
+		objInfo, err := obj.GetObjectInfo("bucket", key)
+		c.Assert(err, check.IsNil)
+		c.Assert(objInfo.Size, check.Equals, int64(len(value)))
+	}
+}
+
+// Tests validate creation of objects and the order of listing using various filters for ListObjects operation.
+func testPaging(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	obj.MakeBucket("bucket")
+	result, err := obj.ListObjects("bucket", "", "", "", 0)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(result.Objects), check.Equals, 0)
+	c.Assert(result.IsTruncated, check.Equals, false)
+	// check before paging occurs.
+	for i := 0; i < 5; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = obj.PutObject("bucket", key, int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
+		c.Assert(err, check.IsNil)
+
+		result, err = obj.ListObjects("bucket", "", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, i+1)
+		c.Assert(result.IsTruncated, check.Equals, false)
+	}
+	// check after paging occurs pages work.
+	for i := 6; i <= 10; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = obj.PutObject("bucket", key, int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
+		c.Assert(err, check.IsNil)
+		result, err = obj.ListObjects("bucket", "obj", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 5)
+		c.Assert(result.IsTruncated, check.Equals, true)
+	}
+	// check paging with prefix at end returns less objects.
+	{
+		_, err = obj.PutObject("bucket", "newPrefix", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
+		c.Assert(err, check.IsNil)
+		_, err = obj.PutObject("bucket", "newPrefix2", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
+		c.Assert(err, check.IsNil)
+		result, err = obj.ListObjects("bucket", "new", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 2)
+	}
+
+	// check ordering of pages.
+	{
+		result, err = obj.ListObjects("bucket", "", "", "", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
+	}
+
+	// check delimited results with delimiter and prefix.
+	{
+		_, err = obj.PutObject("bucket", "this/is/delimited", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
+		c.Assert(err, check.IsNil)
+		_, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
+		c.Assert(err, check.IsNil)
+		result, err = obj.ListObjects("bucket", "this/is/", "", "/", 10)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 1)
+		c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
+	}
+
+	// check delimited results with delimiter without prefix.
+	{
+		result, err = obj.ListObjects("bucket", "", "", "/", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
+		c.Assert(result.Prefixes[0], check.Equals, "this/")
+	}
+
+	// check results with Marker.
+	{
+
+		result, err = obj.ListObjects("bucket", "", "newPrefix", "", 3)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[1].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj1")
+
+	}
+	// check ordering of results with prefix.
+	{
+		result, err = obj.ListObjects("bucket", "obj", "", "", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[1].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj10")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj2")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj3")
+	}
+	// check ordering of results with prefix and no paging.
+	{
+		result, err = obj.ListObjects("bucket", "new", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+	}
+}
+
+// Tests validate overwriting of an existing object.
+func testObjectOverwriteWorks(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	err := obj.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	_, err = obj.PutObject("bucket", "object", int64(len("The list of parts was not in ascending order. The parts list must be specified in order by part number.")), bytes.NewBufferString("The list of parts was not in ascending order. The parts list must be specified in order by part number."), nil)
+	c.Assert(err, check.IsNil)
+
+	length := int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."))
+	_, err = obj.PutObject("bucket", "object", length, bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
+	c.Assert(err, check.IsNil)
+
+	var bytesBuffer bytes.Buffer
+	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer)
+	c.Assert(err, check.IsNil)
+	c.Assert(string(bytesBuffer.Bytes()), check.Equals, "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")
+}
+
+// Tests validate that bucket operation on non-existent bucket fails.
+func testNonExistantBucketOperations(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	_, err := obj.PutObject("bucket1", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(err.Error(), check.Equals, "Bucket not found: bucket1")
+}
+
+// Tests validate that recreation of the bucket fails.
+func testBucketRecreateFails(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	err := obj.MakeBucket("string")
+	c.Assert(err, check.IsNil)
+	err = obj.MakeBucket("string")
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(err.Error(), check.Equals, "Bucket exists: string")
+}
+
+// Tests validate PutObject with subdirectory prefix.
+func testPutObjectInSubdir(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	err := obj.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	length := int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."))
+	_, err = obj.PutObject("bucket", "dir1/dir2/object", length, bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
+	c.Assert(err, check.IsNil)
+
+	var bytesBuffer bytes.Buffer
+	err = obj.GetObject("bucket", "dir1/dir2/object", 0, length, &bytesBuffer)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."))
+}
+
+// Tests validate ListBuckets.
+func testListBuckets(c *check.C, create func() ObjectLayer) {
+	obj := create()
+
+	// test empty list.
+	buckets, err := obj.ListBuckets()
+	c.Assert(err, check.IsNil)
+	c.Assert(len(buckets), check.Equals, 0)
+
+	// add one and test exists.
+	err = obj.MakeBucket("bucket1")
+	c.Assert(err, check.IsNil)
+
+	buckets, err = obj.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 1)
+	c.Assert(err, check.IsNil)
+
+	// add two and test exists.
+	err = obj.MakeBucket("bucket2")
+	c.Assert(err, check.IsNil)
+
+	buckets, err = obj.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 2)
+	c.Assert(err, check.IsNil)
+
+	// add three and test exists + prefix.
+	err = obj.MakeBucket("bucket22")
+
+	buckets, err = obj.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 3)
+	c.Assert(err, check.IsNil)
+}
+
+// Tests validate the order of result of ListBuckets.
+func testListBucketsOrder(c *check.C, create func() ObjectLayer) {
+	// if implementation contains a map, order of map keys will vary.
+	// this ensures they return in the same order each time.
+	for i := 0; i < 10; i++ {
+		obj := create()
+		// add one and test exists.
+		err := obj.MakeBucket("bucket1")
+		c.Assert(err, check.IsNil)
+		err = obj.MakeBucket("bucket2")
+		c.Assert(err, check.IsNil)
+		buckets, err := obj.ListBuckets()
+		c.Assert(err, check.IsNil)
+		c.Assert(len(buckets), check.Equals, 2)
+		c.Assert(buckets[0].Name, check.Equals, "bucket1")
+		c.Assert(buckets[1].Name, check.Equals, "bucket2")
+	}
+}
+
+// Tests validate that ListObjects operation on a non-existent bucket fails as expected.
+func testListObjectsTestsForNonExistantBucket(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	result, err := obj.ListObjects("bucket", "", "", "", 1000)
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(result.IsTruncated, check.Equals, false)
+	c.Assert(len(result.Objects), check.Equals, 0)
+	c.Assert(err.Error(), check.Equals, "Bucket not found: bucket")
+}
+
+// Tests validate that GetObject fails on a non-existent bucket as expected.
+func testNonExistantObjectInBucket(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	err := obj.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	var bytesBuffer bytes.Buffer
+	err = obj.GetObject("bucket", "dir1", 0, 10, &bytesBuffer)
+	c.Assert(err, check.Not(check.IsNil))
+	switch err := err.(type) {
+	case ObjectNotFound:
+		c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
+	default:
+		c.Assert(err, check.Equals, "fails")
+	}
+}
+
+// Tests validate that GetObject on an existing directory fails as expected.
+func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	err := obj.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	_, err = obj.PutObject("bucket", "dir1/dir3/object", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."), nil)
+	c.Assert(err, check.IsNil)
+
+	var bytesBuffer bytes.Buffer
+	err = obj.GetObject("bucket", "dir1", 0, 10, &bytesBuffer)
+	switch err := err.(type) {
+	case ObjectNotFound:
+		c.Assert(err.Bucket, check.Equals, "bucket")
+		c.Assert(err.Object, check.Equals, "dir1")
+	default:
+		// force a failure with a line number.
+		c.Assert(err, check.Equals, "ObjectNotFound")
+	}
+
+	err = obj.GetObject("bucket", "dir1/", 0, 10, &bytesBuffer)
+	switch err := err.(type) {
+	case ObjectNameInvalid:
+		c.Assert(err.Bucket, check.Equals, "bucket")
+		c.Assert(err.Object, check.Equals, "dir1/")
+	default:
+		// force a failure with a line number.
+		c.Assert(err, check.Equals, "ObjectNotFound")
+	}
+}
+
+// Tests valdiate the default ContentType.
+func testDefaultContentType(c *check.C, create func() ObjectLayer) {
+	obj := create()
+	err := obj.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	// Test empty.
+	_, err = obj.PutObject("bucket", "one", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
+	c.Assert(err, check.IsNil)
+	objInfo, err := obj.GetObjectInfo("bucket", "one")
+	c.Assert(err, check.IsNil)
+	c.Assert(objInfo.ContentType, check.Equals, "application/octet-stream")
+}

pkg/atomic/atomic.go

@@ -1,95 +0,0 @@
-/*
- * Minio Cloud Storage (C) 2015-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// NOTE - Rename() not guaranteed to be atomic on all filesystems which are not fully POSIX compatible
-
-// Package atomic provides atomic file write semantics by leveraging Rename's() atomicity.
-package atomic
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-)
-
-// File container provided for atomic file writes
-type File struct {
-	*os.File
-	file string
-}
-
-// CloseAndSync sync file to disk and close, returns an error if any
-func (f *File) CloseAndSync() error {
-	// sync to the disk
-	if err := f.File.Sync(); err != nil {
-		return err
-	}
-	if err := f.Close(); err != nil {
-		return err
-	}
-	return nil
-}
-
-// Close the file, returns an error if any
-func (f *File) Close() error {
-	// close the embedded fd
-	if err := f.File.Close(); err != nil {
-		return err
-	}
-	// atomic rename to final destination
-	if err := os.Rename(f.Name(), f.file); err != nil {
-		return err
-	}
-	return nil
-}
-
-// CloseAndPurge removes the temp file, closes the transaction and returns an error if any
-func (f *File) CloseAndPurge() error {
-	// close the embedded fd
-	if err := f.File.Close(); err != nil {
-		return err
-	}
-	if err := os.Remove(f.Name()); err != nil {
-		return err
-	}
-	return nil
-}
-
-// FileCreate creates a new file at filePath for atomic writes, it also creates parent directories if they don't exist
-func FileCreate(filePath string) (*File, error) {
-	return FileCreateWithPrefix(filePath, "$deleteme.")
-}
-
-// FileCreateWithPrefix creates a new file at filePath for atomic writes, it also creates parent directories if they don't exist
-// prefix specifies the prefix of the temporary files so that cleaning stale temp files is easy
-func FileCreateWithPrefix(filePath string, prefix string) (*File, error) {
-	// if parent directories do not exist, ioutil.TempFile doesn't create them
-	// handle such a case with os.MkdirAll()
-	if err := os.MkdirAll(filepath.Dir(filePath), 0700); err != nil {
-		return nil, err
-	}
-	f, err := ioutil.TempFile(filepath.Dir(filePath), prefix+filepath.Base(filePath))
-	if err != nil {
-		return nil, err
-	}
-	if err = os.Chmod(f.Name(), 0600); err != nil {
-		if err = os.Remove(f.Name()); err != nil {
-			return nil, err
-		}
-		return nil, err
-	}
-	return &File{File: f, file: filePath}, nil
-}

pkg/crypto/sha1/.gitignore

@@ -1 +0,0 @@
-*.syso

pkg/crypto/sha1/sha1.go

@@ -1,150 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file of
-// Golang project:
-//    https://github.com/golang/go/blob/master/LICENSE
-
-// Using this part of Minio codebase under the license
-// Apache License Version 2.0 with modifications
-
-// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174.
-package sha1
-
-import "hash"
-
-// Size - The size of a SHA1 checksum in bytes.
-const Size = 20
-
-// BlockSize - The blocksize of SHA1 in bytes.
-const BlockSize = 64
-
-const (
-	chunk = 64
-	init0 = 0x67452301
-	init1 = 0xEFCDAB89
-	init2 = 0x98BADCFE
-	init3 = 0x10325476
-	init4 = 0xC3D2E1F0
-)
-
-// digest represents the partial evaluation of a checksum.
-type digest struct {
-	h   [5]uint32
-	x   [chunk]byte
-	nx  int
-	len uint64
-}
-
-// Reset digest
-func (d *digest) Reset() {
-	d.h[0] = init0
-	d.h[1] = init1
-	d.h[2] = init2
-	d.h[3] = init3
-	d.h[4] = init4
-	d.nx = 0
-	d.len = 0
-}
-
-// New returns a new hash.Hash computing the SHA1 checksum.
-func New() hash.Hash {
-	d := new(digest)
-	d.Reset()
-	return d
-}
-
-// Return output size
-func (d *digest) Size() int { return Size }
-
-// Return checksum blocksize
-func (d *digest) BlockSize() int { return BlockSize }
-
-// Write to digest
-func (d *digest) Write(p []byte) (nn int, err error) {
-	nn = len(p)
-	d.len += uint64(nn)
-	if d.nx > 0 {
-		n := copy(d.x[d.nx:], p)
-		d.nx += n
-		if d.nx == chunk {
-			block(d, d.x[:])
-			d.nx = 0
-		}
-		p = p[n:]
-	}
-	if len(p) >= chunk {
-		n := len(p) &^ (chunk - 1)
-		block(d, p[:n])
-		p = p[n:]
-	}
-	if len(p) > 0 {
-		d.nx = copy(d.x[:], p)
-	}
-	return
-}
-
-// Return checksum bytes
-func (d *digest) Sum(in []byte) []byte {
-	// Make a copy of d0 so that caller can keep writing and summing.
-	d0 := *d
-	hash := d0.checkSum()
-	return append(in, hash[:]...)
-}
-
-// Intermediate checksum function
-func (d *digest) checkSum() [Size]byte {
-	len := d.len
-	// Padding.  Add a 1 bit and 0 bits until 56 bytes mod 64.
-	var tmp [64]byte
-	tmp[0] = 0x80
-	if len%64 < 56 {
-		d.Write(tmp[0 : 56-len%64])
-	} else {
-		d.Write(tmp[0 : 64+56-len%64])
-	}
-
-	// Length in bits.
-	len <<= 3
-	for i := uint(0); i < 8; i++ {
-		tmp[i] = byte(len >> (56 - 8*i))
-	}
-	d.Write(tmp[0:8])
-
-	if d.nx != 0 {
-		panic("d.nx != 0")
-	}
-
-	var digest [Size]byte
-	for i, s := range d.h {
-		digest[i*4] = byte(s >> 24)
-		digest[i*4+1] = byte(s >> 16)
-		digest[i*4+2] = byte(s >> 8)
-		digest[i*4+3] = byte(s)
-	}
-
-	return digest
-}
-
-// Sum - single caller sha1 helper
-func Sum(data []byte) [Size]byte {
-	var d digest
-	d.Reset()
-	d.Write(data)
-	return d.checkSum()
-}

pkg/crypto/sha1/sha1_linux.S

@@ -1,967 +0,0 @@
-/*
- *	Implement fast SHA-1 with AVX2 instructions. (x86_64)
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- * Ilya Albrekht <ilya.albrekht@intel.com>
- * Maxim Locktyukhin <maxim.locktyukhin@intel.com>
- * Ronen Zohar <ronen.zohar@intel.com>
- * Chandramouli Narayanan <mouli@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2014 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * SHA-1 implementation with Intel(R) AVX2 instruction set extensions.
- *
- *This implementation is based on the previous SSSE3 release:
- * https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
- *
- *Updates 20-byte SHA-1 record in 'hash' for even number of
- *'num_blocks' consecutive 64-byte blocks
- *
- */
-
-/*
- * Using this part of Minio codebase under the license
- * Apache License Version 2.0 with modifications
- *
- */
-
-#ifdef HAS_AVX2
-#ifndef ENTRY
-#define ENTRY(name) \
-        .globl name             ; \
-        .align 4,0x90           ; \
-        name:
-#endif
-
-#ifndef END
-#define END(name) \
-        .size name, .-name
-#endif
-
-#ifndef ENDPROC
-#define ENDPROC(name) \
-        .type name, @function   ; \
-        END(name)
-#endif
-
-#define NUM_INVALID		100
-
-#define TYPE_R32		0
-#define TYPE_R64		1
-#define TYPE_XMM		2
-#define TYPE_INVALID	100
-
-	.macro R32_NUM opd r32
-	\opd = NUM_INVALID
-	.ifc \r32,%eax
-	\opd = 0
-	.endif
-	.ifc \r32,%ecx
-	\opd = 1
-	.endif
-	.ifc \r32,%edx
-	\opd = 2
-	.endif
-	.ifc \r32,%ebx
-	\opd = 3
-	.endif
-	.ifc \r32,%esp
-	\opd = 4
-	.endif
-	.ifc \r32,%ebp
-	\opd = 5
-	.endif
-	.ifc \r32,%esi
-	\opd = 6
-	.endif
-	.ifc \r32,%edi
-	\opd = 7
-	.endif
-#ifdef X86_64
-	.ifc \r32,%r8d
-	\opd = 8
-	.endif
-	.ifc \r32,%r9d
-	\opd = 9
-	.endif
-	.ifc \r32,%r10d
-	\opd = 10
-	.endif
-	.ifc \r32,%r11d
-	\opd = 11
-	.endif
-	.ifc \r32,%r12d
-	\opd = 12
-	.endif
-	.ifc \r32,%r13d
-	\opd = 13
-	.endif
-	.ifc \r32,%r14d
-	\opd = 14
-	.endif
-	.ifc \r32,%r15d
-	\opd = 15
-	.endif
-#endif
-	.endm
-
-	.macro R64_NUM opd r64
-	\opd = NUM_INVALID
-#ifdef X86_64
-	.ifc \r64,%rax
-	\opd = 0
-	.endif
-	.ifc \r64,%rcx
-	\opd = 1
-	.endif
-	.ifc \r64,%rdx
-	\opd = 2
-	.endif
-	.ifc \r64,%rbx
-	\opd = 3
-	.endif
-	.ifc \r64,%rsp
-	\opd = 4
-	.endif
-	.ifc \r64,%rbp
-	\opd = 5
-	.endif
-	.ifc \r64,%rsi
-	\opd = 6
-	.endif
-	.ifc \r64,%rdi
-	\opd = 7
-	.endif
-	.ifc \r64,%r8
-	\opd = 8
-	.endif
-	.ifc \r64,%r9
-	\opd = 9
-	.endif
-	.ifc \r64,%r10
-	\opd = 10
-	.endif
-	.ifc \r64,%r11
-	\opd = 11
-	.endif
-	.ifc \r64,%r12
-	\opd = 12
-	.endif
-	.ifc \r64,%r13
-	\opd = 13
-	.endif
-	.ifc \r64,%r14
-	\opd = 14
-	.endif
-	.ifc \r64,%r15
-	\opd = 15
-	.endif
-#endif
-	.endm
-
-	.macro XMM_NUM opd xmm
-	\opd = NUM_INVALID
-	.ifc \xmm,%xmm0
-	\opd = 0
-	.endif
-	.ifc \xmm,%xmm1
-	\opd = 1
-	.endif
-	.ifc \xmm,%xmm2
-	\opd = 2
-	.endif
-	.ifc \xmm,%xmm3
-	\opd = 3
-	.endif
-	.ifc \xmm,%xmm4
-	\opd = 4
-	.endif
-	.ifc \xmm,%xmm5
-	\opd = 5
-	.endif
-	.ifc \xmm,%xmm6
-	\opd = 6
-	.endif
-	.ifc \xmm,%xmm7
-	\opd = 7
-	.endif
-	.ifc \xmm,%xmm8
-	\opd = 8
-	.endif
-	.ifc \xmm,%xmm9
-	\opd = 9
-	.endif
-	.ifc \xmm,%xmm10
-	\opd = 10
-	.endif
-	.ifc \xmm,%xmm11
-	\opd = 11
-	.endif
-	.ifc \xmm,%xmm12
-	\opd = 12
-	.endif
-	.ifc \xmm,%xmm13
-	\opd = 13
-	.endif
-	.ifc \xmm,%xmm14
-	\opd = 14
-	.endif
-	.ifc \xmm,%xmm15
-	\opd = 15
-	.endif
-	.endm
-
-	.macro TYPE type reg
-	R32_NUM reg_type_r32 \reg
-	R64_NUM reg_type_r64 \reg
-	XMM_NUM reg_type_xmm \reg
-	.if reg_type_r64 <> NUM_INVALID
-	\type = TYPE_R64
-	.elseif reg_type_r32 <> NUM_INVALID
-	\type = TYPE_R32
-	.elseif reg_type_xmm <> NUM_INVALID
-	\type = TYPE_XMM
-	.else
-	\type = TYPE_INVALID
-	.endif
-	.endm
-
-	.macro PFX_OPD_SIZE
-	.byte 0x66
-	.endm
-
-	.macro PFX_REX opd1 opd2 W=0
-	.if ((\opd1 | \opd2) & 8) || \W
-	.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
-	.endif
-	.endm
-
-	.macro MODRM mod opd1 opd2
-	.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
-	.endm
-
-	.macro PSHUFB_XMM xmm1 xmm2
-	XMM_NUM pshufb_opd1 \xmm1
-	XMM_NUM pshufb_opd2 \xmm2
-	PFX_OPD_SIZE
-	PFX_REX pshufb_opd1 pshufb_opd2
-	.byte 0x0f, 0x38, 0x00
-	MODRM 0xc0 pshufb_opd1 pshufb_opd2
-	.endm
-
-	.macro PCLMULQDQ imm8 xmm1 xmm2
-	XMM_NUM clmul_opd1 \xmm1
-	XMM_NUM clmul_opd2 \xmm2
-	PFX_OPD_SIZE
-	PFX_REX clmul_opd1 clmul_opd2
-	.byte 0x0f, 0x3a, 0x44
-	MODRM 0xc0 clmul_opd1 clmul_opd2
-	.byte \imm8
-	.endm
-
-	.macro PEXTRD imm8 xmm gpr
-	R32_NUM extrd_opd1 \gpr
-	XMM_NUM extrd_opd2 \xmm
-	PFX_OPD_SIZE
-	PFX_REX extrd_opd1 extrd_opd2
-	.byte 0x0f, 0x3a, 0x16
-	MODRM 0xc0 extrd_opd1 extrd_opd2
-	.byte \imm8
-	.endm
-
-	.macro MOVQ_R64_XMM opd1 opd2
-	TYPE movq_r64_xmm_opd1_type \opd1
-	.if movq_r64_xmm_opd1_type == TYPE_XMM
-	XMM_NUM movq_r64_xmm_opd1 \opd1
-	R64_NUM movq_r64_xmm_opd2 \opd2
-	.else
-	R64_NUM movq_r64_xmm_opd1 \opd1
-	XMM_NUM movq_r64_xmm_opd2 \opd2
-	.endif
-	PFX_OPD_SIZE
-	PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
-	.if movq_r64_xmm_opd1_type == TYPE_XMM
-	.byte 0x0f, 0x7e
-	.else
-	.byte 0x0f, 0x6e
-	.endif
-	MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
-	.endm
-
-#define	CTX	%rdi	/* arg1 */
-#define BUF	%rsi	/* arg2 */
-#define CNT	%rdx	/* arg3 */
-
-#define	REG_A	%ecx
-#define	REG_B	%esi
-#define	REG_C	%edi
-#define	REG_D	%eax
-#define	REG_E	%edx
-#define	REG_TB	%ebx
-#define	REG_TA	%r12d
-#define	REG_RA	%rcx
-#define	REG_RB	%rsi
-#define	REG_RC	%rdi
-#define	REG_RD	%rax
-#define	REG_RE	%rdx
-#define	REG_RTA	%r12
-#define	REG_RTB	%rbx
-#define	REG_T1	%ebp
-#define	xmm_mov	vmovups
-#define	avx2_zeroupper	vzeroupper
-#define	RND_F1	1
-#define	RND_F2	2
-#define	RND_F3	3
-
-.macro REGALLOC
-	.set A, REG_A
-	.set B, REG_B
-	.set C, REG_C
-	.set D, REG_D
-	.set E, REG_E
-	.set TB, REG_TB
-	.set TA, REG_TA
-
-	.set RA, REG_RA
-	.set RB, REG_RB
-	.set RC, REG_RC
-	.set RD, REG_RD
-	.set RE, REG_RE
-
-	.set RTA, REG_RTA
-	.set RTB, REG_RTB
-
-	.set T1, REG_T1
-.endm
-
-#define K_BASE		%r8
-#define HASH_PTR	%r9
-#define BUFFER_PTR	%r10
-#define BUFFER_PTR2	%r13
-#define BUFFER_END	%r11
-
-#define PRECALC_BUF	%r14
-#define WK_BUF		%r15
-
-#define W_TMP		%xmm0
-#define WY_TMP		%ymm0
-#define WY_TMP2		%ymm9
-
-# AVX2 variables
-#define WY0		%ymm3
-#define WY4		%ymm5
-#define WY08		%ymm7
-#define WY12		%ymm8
-#define WY16		%ymm12
-#define WY20		%ymm13
-#define WY24		%ymm14
-#define WY28		%ymm15
-
-#define YMM_SHUFB_BSWAP	%ymm10
-
-/*
- * Keep 2 iterations precalculated at a time:
- *    - 80 DWORDs per iteration * 2
- */
-#define W_SIZE		(80*2*2 +16)
-
-#define WK(t)	((((t) % 80) / 4)*32 + ( (t) % 4)*4 + ((t)/80)*16 )(WK_BUF)
-#define PRECALC_WK(t)	((t)*2*2)(PRECALC_BUF)
-
-
-.macro UPDATE_HASH  hash, val
-	add	\hash, \val
-	mov	\val, \hash
-.endm
-
-.macro PRECALC_RESET_WY
-	.set WY_00, WY0
-	.set WY_04, WY4
-	.set WY_08, WY08
-	.set WY_12, WY12
-	.set WY_16, WY16
-	.set WY_20, WY20
-	.set WY_24, WY24
-	.set WY_28, WY28
-	.set WY_32, WY_00
-.endm
-
-.macro PRECALC_ROTATE_WY
-	/* Rotate macros */
-	.set WY_32, WY_28
-	.set WY_28, WY_24
-	.set WY_24, WY_20
-	.set WY_20, WY_16
-	.set WY_16, WY_12
-	.set WY_12, WY_08
-	.set WY_08, WY_04
-	.set WY_04, WY_00
-	.set WY_00, WY_32
-
-	/* Define register aliases */
-	.set WY, WY_00
-	.set WY_minus_04, WY_04
-	.set WY_minus_08, WY_08
-	.set WY_minus_12, WY_12
-	.set WY_minus_16, WY_16
-	.set WY_minus_20, WY_20
-	.set WY_minus_24, WY_24
-	.set WY_minus_28, WY_28
-	.set WY_minus_32, WY
-.endm
-
-.macro PRECALC_00_15
-	.if (i == 0) # Initialize and rotate registers
-		PRECALC_RESET_WY
-		PRECALC_ROTATE_WY
-	.endif
-
-	/* message scheduling pre-compute for rounds 0-15 */
-	.if   ((i & 7) == 0)
-		/*
-		 * blended AVX2 and ALU instruction scheduling
-		 * 1 vector iteration per 8 rounds
-		 */
-		vmovdqu ((i * 2) + PRECALC_OFFSET)(BUFFER_PTR), W_TMP
-	.elseif ((i & 7) == 1)
-		vinsertf128 $1, (((i-1) * 2)+PRECALC_OFFSET)(BUFFER_PTR2),\
-			 WY_TMP, WY_TMP
-	.elseif ((i & 7) == 2)
-		vpshufb YMM_SHUFB_BSWAP, WY_TMP, WY
-	.elseif ((i & 7) == 4)
-		vpaddd  K_XMM(K_BASE), WY, WY_TMP
-	.elseif ((i & 7) == 7)
-		vmovdqu  WY_TMP, PRECALC_WK(i&~7)
-
-		PRECALC_ROTATE_WY
-	.endif
-.endm
-
-.macro PRECALC_16_31
-	/*
-	 * message scheduling pre-compute for rounds 16-31
-	 * calculating last 32 w[i] values in 8 XMM registers
-	 * pre-calculate K+w[i] values and store to mem
-	 * for later load by ALU add instruction
-	 *
-	 * "brute force" vectorization for rounds 16-31 only
-	 * due to w[i]->w[i-3] dependency
-	 */
-	.if   ((i & 7) == 0)
-		/*
-		 * blended AVX2 and ALU instruction scheduling
-		 * 1 vector iteration per 8 rounds
-		 */
-		/* w[i-14] */
-		vpalignr	$8, WY_minus_16, WY_minus_12, WY
-		vpsrldq	$4, WY_minus_04, WY_TMP               /* w[i-3] */
-	.elseif ((i & 7) == 1)
-		vpxor	WY_minus_08, WY, WY
-		vpxor	WY_minus_16, WY_TMP, WY_TMP
-	.elseif ((i & 7) == 2)
-		vpxor	WY_TMP, WY, WY
-		vpslldq	$12, WY, WY_TMP2
-	.elseif ((i & 7) == 3)
-		vpslld	$1, WY, WY_TMP
-		vpsrld	$31, WY, WY
-	.elseif ((i & 7) == 4)
-		vpor	WY, WY_TMP, WY_TMP
-		vpslld	$2, WY_TMP2, WY
-	.elseif ((i & 7) == 5)
-		vpsrld	$30, WY_TMP2, WY_TMP2
-		vpxor	WY, WY_TMP, WY_TMP
-	.elseif ((i & 7) == 7)
-		vpxor	WY_TMP2, WY_TMP, WY
-		vpaddd	K_XMM(K_BASE), WY, WY_TMP
-		vmovdqu	WY_TMP, PRECALC_WK(i&~7)
-
-		PRECALC_ROTATE_WY
-	.endif
-.endm
-
-.macro PRECALC_32_79
-	/*
-	 * in SHA-1 specification:
-	 * w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
-	 * instead we do equal:
-	 * w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
-	 * allows more efficient vectorization
-	 * since w[i]=>w[i-3] dependency is broken
-	 */
-
-	.if   ((i & 7) == 0)
-	/*
-	 * blended AVX2 and ALU instruction scheduling
-	 * 1 vector iteration per 8 rounds
-	 */
-		vpalignr	$8, WY_minus_08, WY_minus_04, WY_TMP
-	.elseif ((i & 7) == 1)
-		/* W is W_minus_32 before xor */
-		vpxor	WY_minus_28, WY, WY
-	.elseif ((i & 7) == 2)
-		vpxor	WY_minus_16, WY_TMP, WY_TMP
-	.elseif ((i & 7) == 3)
-		vpxor	WY_TMP, WY, WY
-	.elseif ((i & 7) == 4)
-		vpslld	$2, WY, WY_TMP
-	.elseif ((i & 7) == 5)
-		vpsrld	$30, WY, WY
-		vpor	WY, WY_TMP, WY
-	.elseif ((i & 7) == 7)
-		vpaddd	K_XMM(K_BASE), WY, WY_TMP
-		vmovdqu	WY_TMP, PRECALC_WK(i&~7)
-
-		PRECALC_ROTATE_WY
-	.endif
-.endm
-
-.macro PRECALC r, s
-	.set i, \r
-
-	.if (i < 40)
-		.set K_XMM, 32*0
-	.elseif (i < 80)
-		.set K_XMM, 32*1
-	.elseif (i < 120)
-		.set K_XMM, 32*2
-	.else
-		.set K_XMM, 32*3
-	.endif
-
-	.if (i<32)
-		PRECALC_00_15	\s
-	.elseif (i<64)
-		PRECALC_16_31	\s
-	.elseif (i < 160)
-		PRECALC_32_79	\s
-	.endif
-.endm
-
-.macro ROTATE_STATE
-	.set T_REG, E
-	.set E, D
-	.set D, C
-	.set C, B
-	.set B, TB
-	.set TB, A
-	.set A, T_REG
-
-	.set T_REG, RE
-	.set RE, RD
-	.set RD, RC
-	.set RC, RB
-	.set RB, RTB
-	.set RTB, RA
-	.set RA, T_REG
-.endm
-
-/* Macro relies on saved ROUND_Fx */
-
-.macro RND_FUN f, r
-	.if (\f == RND_F1)
-		ROUND_F1	\r
-	.elseif (\f == RND_F2)
-		ROUND_F2	\r
-	.elseif (\f == RND_F3)
-		ROUND_F3	\r
-	.endif
-.endm
-
-.macro RR r
-	.set round_id, (\r % 80)
-
-	.if (round_id == 0)        /* Precalculate F for first round */
-		.set ROUND_FUNC, RND_F1
-		mov	B, TB
-
-		rorx	$(32-30), B, B    /* b>>>2 */
-		andn	D, TB, T1
-		and	C, TB
-		xor	T1, TB
-	.endif
-
-	RND_FUN ROUND_FUNC, \r
-	ROTATE_STATE
-
-	.if   (round_id == 18)
-		.set ROUND_FUNC, RND_F2
-	.elseif (round_id == 38)
-		.set ROUND_FUNC, RND_F3
-	.elseif (round_id == 58)
-		.set ROUND_FUNC, RND_F2
-	.endif
-
-	.set round_id, ( (\r+1) % 80)
-
-	RND_FUN ROUND_FUNC, (\r+1)
-	ROTATE_STATE
-.endm
-
-.macro ROUND_F1 r
-	add	WK(\r), E
-
-	andn	C, A, T1			/* ~b&d */
-	lea	(RE,RTB), E		/* Add F from the previous round */
-
-	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
-	rorx	$(32-30),A, TB		/* b>>>2 for next round */
-
-	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
-
-	/*
-	 * Calculate F for the next round
-	 * (b & c) ^ andn[b, d]
-	 */
-	and	B, A			/* b&c */
-	xor	T1, A			/* F1 = (b&c) ^ (~b&d) */
-
-	lea	(RE,RTA), E		/* E += A >>> 5 */
-.endm
-
-.macro ROUND_F2 r
-	add	WK(\r), E
-	lea	(RE,RTB), E		/* Add F from the previous round */
-
-	/* Calculate F for the next round */
-	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
-	.if ((round_id) < 79)
-		rorx	$(32-30), A, TB	/* b>>>2 for next round */
-	.endif
-	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
-
-	.if ((round_id) < 79)
-		xor	B, A
-	.endif
-
-	add	TA, E			/* E += A >>> 5 */
-
-	.if ((round_id) < 79)
-		xor	C, A
-	.endif
-.endm
-
-.macro ROUND_F3 r
-	add	WK(\r), E
-	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
-
-	lea	(RE,RTB), E		/* Add F from the previous round */
-
-	mov	B, T1
-	or	A, T1
-
-	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
-	rorx	$(32-30), A, TB		/* b>>>2 for next round */
-
-	/* Calculate F for the next round
-	 * (b and c) or (d and (b or c))
-	 */
-	and	C, T1
-	and	B, A
-	or	T1, A
-
-	add	TA, E			/* E += A >>> 5 */
-
-.endm
-
-/*
- * macro implements 80 rounds of SHA-1, for multiple blocks with s/w pipelining
- */
-.macro SHA1_PIPELINED_MAIN_BODY
-
-	REGALLOC
-
-	mov	(HASH_PTR), A
-	mov	4(HASH_PTR), B
-	mov	8(HASH_PTR), C
-	mov	12(HASH_PTR), D
-	mov	16(HASH_PTR), E
-
-	mov	%rsp, PRECALC_BUF
-	lea	(2*4*80+32)(%rsp), WK_BUF
-
-	# Precalc WK for first 2 blocks
-	PRECALC_OFFSET = 0
-	.set i, 0
-	.rept    160
-		PRECALC i
-		.set i, i + 1
-	.endr
-	PRECALC_OFFSET = 128
-	xchg	WK_BUF, PRECALC_BUF
-
-	.align 32
-_loop:
-	/*
-	 * code loops through more than one block
-	 * we use K_BASE value as a signal of a last block,
-	 * it is set below by: cmovae BUFFER_PTR, K_BASE
-	 */
-	cmp	K_BASE, BUFFER_PTR
-	jne	_begin
-	.align 32
-	jmp	_end
-	.align 32
-_begin:
-
-	/*
-	 * Do first block
-	 * rounds: 0,2,4,6,8
-	 */
-	.set j, 0
-	.rept 5
-		RR	j
-		.set j, j+2
-	.endr
-
-	jmp _loop0
-_loop0:
-
-	/*
-	 * rounds:
-	 * 10,12,14,16,18
-	 * 20,22,24,26,28
-	 * 30,32,34,36,38
-	 * 40,42,44,46,48
-	 * 50,52,54,56,58
-	 */
-	.rept 25
-		RR	j
-		.set j, j+2
-	.endr
-
-	add	$(2*64), BUFFER_PTR       /* move to next odd-64-byte block */
-	cmp	BUFFER_END, BUFFER_PTR    /* is current block the last one? */
-	cmovae	K_BASE, BUFFER_PTR	/* signal the last iteration smartly */
-
-	/*
-	 * rounds
-	 * 60,62,64,66,68
-	 * 70,72,74,76,78
-	 */
-	.rept 10
-		RR	j
-		.set j, j+2
-	.endr
-
-	UPDATE_HASH	(HASH_PTR), A
-	UPDATE_HASH	4(HASH_PTR), TB
-	UPDATE_HASH	8(HASH_PTR), C
-	UPDATE_HASH	12(HASH_PTR), D
-	UPDATE_HASH	16(HASH_PTR), E
-
-	cmp	K_BASE, BUFFER_PTR	/* is current block the last one? */
-	je	_loop
-
-	mov	TB, B
-
-	/* Process second block */
-	/*
-	 * rounds
-	 *  0+80, 2+80, 4+80, 6+80, 8+80
-	 * 10+80,12+80,14+80,16+80,18+80
-	 */
-
-	.set j, 0
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	jmp	_loop1
-_loop1:
-	/*
-	 * rounds
-	 * 20+80,22+80,24+80,26+80,28+80
-	 * 30+80,32+80,34+80,36+80,38+80
-	 */
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	jmp	_loop2
-_loop2:
-
-	/*
-	 * rounds
-	 * 40+80,42+80,44+80,46+80,48+80
-	 * 50+80,52+80,54+80,56+80,58+80
-	 */
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	add	$(2*64), BUFFER_PTR2      /* move to next even-64-byte block */
-
-	cmp	BUFFER_END, BUFFER_PTR2   /* is current block the last one */
-	cmovae	K_BASE, BUFFER_PTR       /* signal the last iteration smartly */
-
-	jmp	_loop3
-_loop3:
-
-	/*
-	 * rounds
-	 * 60+80,62+80,64+80,66+80,68+80
-	 * 70+80,72+80,74+80,76+80,78+80
-	 */
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	UPDATE_HASH	(HASH_PTR), A
-	UPDATE_HASH	4(HASH_PTR), TB
-	UPDATE_HASH	8(HASH_PTR), C
-	UPDATE_HASH	12(HASH_PTR), D
-	UPDATE_HASH	16(HASH_PTR), E
-
-	/* Reset state for AVX2 reg permutation */
-	mov	A, TA
-	mov	TB, A
-	mov	C, TB
-	mov	E, C
-	mov	D, B
-	mov	TA, D
-
-	REGALLOC
-
-	xchg	WK_BUF, PRECALC_BUF
-
-	jmp	_loop
-
-	.align 32
-	_end:
-
-.endm
-
-.section .rodata
-
-#define K1 0x5a827999
-#define K2 0x6ed9eba1
-#define K3 0x8f1bbcdc
-#define K4 0xca62c1d6
-
-.align 128
-K_XMM_AR:
-	.long K1, K1, K1, K1
-	.long K1, K1, K1, K1
-	.long K2, K2, K2, K2
-	.long K2, K2, K2, K2
-	.long K3, K3, K3, K3
-	.long K3, K3, K3, K3
-	.long K4, K4, K4, K4
-	.long K4, K4, K4, K4
-
-BSWAP_SHUFB_CTL:
-	.long 0x00010203
-	.long 0x04050607
-	.long 0x08090a0b
-	.long 0x0c0d0e0f
-	.long 0x00010203
-	.long 0x04050607
-	.long 0x08090a0b
-	.long 0x0c0d0e0f
-
-# void sha1_transform(int32_t *hash, const char* input, size_t num_blocks) ;
-        .text
-	ENTRY(sha1_transform)
-
-	push	%rbx
-	push	%rbp
-	push	%r12
-	push	%r13
-	push	%r14
-	push	%r15
-
-	RESERVE_STACK  = (W_SIZE*4 + 8+24)
-
-	/* Align stack */
-	mov	%rsp, %rbx
-	and	$~(0x20-1), %rsp
-	push	%rbx
-	sub	$RESERVE_STACK, %rsp
-
-	avx2_zeroupper
-
-	lea	K_XMM_AR(%rip), K_BASE
-
-	mov	CTX, HASH_PTR
-	mov	BUF, BUFFER_PTR
-	lea	64(BUF), BUFFER_PTR2
-
-	shl	$6, CNT			/* mul by 64 */
-	add	BUF, CNT
-	add	$64, CNT
-	mov	CNT, BUFFER_END
-
-	cmp	BUFFER_END, BUFFER_PTR2
-	cmovae	K_BASE, BUFFER_PTR2
-
-	xmm_mov	BSWAP_SHUFB_CTL(%rip), YMM_SHUFB_BSWAP
-
-	SHA1_PIPELINED_MAIN_BODY
-
-	avx2_zeroupper
-
-	add	$RESERVE_STACK, %rsp
-	pop	%rsp
-
-	pop	%r15
-	pop	%r14
-	pop	%r13
-	pop	%r12
-	pop	%rbp
-	pop	%rbx
-
-	ret
-
-	ENDPROC(sha1_transform)
-#endif

pkg/crypto/sha1/sha1_sse3_amd64.asm

@@ -1,579 +0,0 @@
-;---------------------
-;   https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
-;
-; License information:
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;  This implementation notably advances the performance of SHA-1 algorithm compared to existing
-;  implementations. We are encouraging all projects utilizing SHA-1 to integrate this new fast
-;  implementation and are ready to help if issues or concerns arise (you are welcome to leave
-;  a comment or write an email to the authors). It is provided 'as is' and free for either
-;  commercial or non-commercial use.
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;
-;   This code implements two interfaces of SHA-1 update function: 1) working on a single
-;   64-byte block and 2) working on a buffer of multiple 64-bit blocks. Multiple blocks
-;   version of code is software pipelined and faster overall, it is a default. Assemble
-;   with -DINTEL_SHA1_SINGLEBLOCK to select single 64-byte block function interface.
-;
-;   C++ prototypes of implemented functions are below:
-;
-;   #ifndef INTEL_SHA1_SINGLEBLOCK
-;      // Updates 20-byte SHA-1 record in 'hash' for 'num_blocks' consequtive 64-byte blocks
-;      extern "C" void sha1_update_intel(int *hash, const char* input, size_t num_blocks );
-;   #else
-;      // Updates 20-byte SHA-1 record in 'hash' for one 64-byte block pointed by 'input'
-;      extern "C" void sha1_update_intel(int *hash, const char* input);
-;   #endif
-;
-;   Function name 'sha1_update_intel' can be changed in the source or via macro:
-;     -DINTEL_SHA1_UPDATE_FUNCNAME=my_sha1_update_func_name
-;
-;   It implements both UNIX(default) and Windows ABIs, use -DWIN_ABI on Windows
-;
-;   Code checks CPU for SSSE3 support via CPUID feature flag (CPUID.1.ECX.SSSE3[bit 9]==1),
-;   and performs dispatch. Since in most cases the functionality on non-SSSE3 supporting CPUs
-;   is also required, the default (e.g. one being replaced) function can be provided for
-;   dispatch on such CPUs, the name of old function can be changed in the source or via macro:
-;      -DINTEL_SHA1_UPDATE_DEFAULT_DISPATCH=default_sha1_update_function_name
-;
-;   Authors: Maxim Locktyukhin and Ronen Zohar at Intel.com
-;
-
-%ifndef INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
- ;; can be replaced with a default SHA-1 update function name
-%define INTEL_SHA1_UPDATE_DEFAULT_DISPATCH  sha1_intel_non_ssse3_cpu_stub_
-%else
-extern  INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
-%endif
-
-;; provide alternative SHA-1 update function's name here
-%ifndef INTEL_SHA1_UPDATE_FUNCNAME
-%define INTEL_SHA1_UPDATE_FUNCNAME     sha1_update_intel
-%endif
-
-global INTEL_SHA1_UPDATE_FUNCNAME
-
-
-%ifndef INTEL_SHA1_SINGLEBLOCK
-%assign multiblock 1
-%else
-%assign multiblock 0
-%endif
-
-
-bits 64
-default rel
-
-%ifdef WIN_ABI
- %xdefine arg1 rcx
- %xdefine arg2 rdx
- %xdefine arg3 r8
-%else
- %xdefine arg1 rdi
- %xdefine arg2 rsi
- %xdefine arg3 rdx
-%endif
-
-%xdefine ctx arg1
-%xdefine buf arg2
-%xdefine cnt arg3
-
-%macro REGALLOC 0
- %xdefine A ecx
- %xdefine B esi
- %xdefine C edi
- %xdefine D ebp
- %xdefine E edx
-
- %xdefine T1 eax
- %xdefine T2 ebx
-%endmacro
-
-%xdefine K_BASE     r8
-%xdefine HASH_PTR   r9
-%xdefine BUFFER_PTR r10
-%xdefine BUFFER_END r11
-
-%xdefine W_TMP  xmm0
-%xdefine W_TMP2 xmm9
-
-%xdefine W0  xmm1
-%xdefine W4  xmm2
-%xdefine W8  xmm3
-%xdefine W12 xmm4
-%xdefine W16 xmm5
-%xdefine W20 xmm6
-%xdefine W24 xmm7
-%xdefine W28 xmm8
-
-%xdefine XMM_SHUFB_BSWAP xmm10
-
-;; we keep window of 64 w[i]+K pre-calculated values in a circular buffer
-%xdefine WK(t) (rsp + (t & 15)*4)
-
-;------------------------------------------------------------------------------
-;
-; macro implements SHA-1 function's body for single or several 64-byte blocks
-; first param: function's name
-; second param: =0 - function implements single 64-byte block hash
-;               =1 - function implements multiple64-byte blocks hash
-;                    3rd function's argument is a number, greater 0, of 64-byte blocks to calc hash for
-;
-%macro  SHA1_VECTOR_ASM  2
-align 4096
-%1:
- push rbx
- push rbp
-
- %ifdef WIN_ABI
- push rdi
- push rsi
-
- %xdefine stack_size (16*4 + 16*5 + 8)
- %else
- %xdefine stack_size (16*4 + 8)
- %endif
-
- sub     rsp, stack_size
-
- %ifdef WIN_ABI
- %xdefine xmm_save_base (rsp + 16*4)
-
- xmm_mov [xmm_save_base + 0*16], xmm6
- xmm_mov [xmm_save_base + 1*16], xmm7
- xmm_mov [xmm_save_base + 2*16], xmm8
- xmm_mov [xmm_save_base + 3*16], xmm9
- xmm_mov [xmm_save_base + 4*16], xmm10
- %endif
-
- mov     HASH_PTR, ctx
- mov     BUFFER_PTR, buf
-
- %if (%2 == 1)
- shl     cnt, 6           ;; mul by 64
- add     cnt, buf
- mov     BUFFER_END, cnt
- %endif
-
- lea     K_BASE, [K_XMM_AR]
- xmm_mov XMM_SHUFB_BSWAP, [bswap_shufb_ctl]
-
- SHA1_PIPELINED_MAIN_BODY %2
-
- %ifdef WIN_ABI
- xmm_mov xmm6, [xmm_save_base + 0*16]
- xmm_mov xmm7, [xmm_save_base + 1*16]
- xmm_mov xmm8, [xmm_save_base + 2*16]
- xmm_mov xmm9, [xmm_save_base + 3*16]
- xmm_mov xmm10,[xmm_save_base + 4*16]
- %endif
-
- add rsp, stack_size
-
- %ifdef WIN_ABI
- pop rsi
- pop rdi
- %endif
-
- pop rbp
- pop rbx
-
- ret
-%endmacro
-
-;--------------------------------------------
-; macro implements 80 rounds of SHA-1, for one 64-byte block or multiple blocks with s/w pipelining
-; macro param: =0 - process single 64-byte block
-;              =1 - multiple blocks
-;
-%macro SHA1_PIPELINED_MAIN_BODY 1
-
- REGALLOC
-
- mov A, [HASH_PTR   ]
- mov B, [HASH_PTR+ 4]
- mov C, [HASH_PTR+ 8]
- mov D, [HASH_PTR+12]
-
- mov E, [HASH_PTR+16]
-
- %assign i 0
- %rep    W_PRECALC_AHEAD
- W_PRECALC i
- %assign i i+1
- %endrep
-
- %xdefine F F1
-
- %if (%1 == 1)                         ;; code loops through more than one block
- %%_loop:
- cmp BUFFER_PTR, K_BASE          ;; we use K_BASE value as a signal of a last block,
- jne %%_begin                    ;; it is set below by: cmovae BUFFER_PTR, K_BASE
- jmp %%_end
-
- align 32
- %%_begin:
- %endif
- RR A,B,C,D,E,0
- RR D,E,A,B,C,2
- RR B,C,D,E,A,4
- RR E,A,B,C,D,6
- RR C,D,E,A,B,8
-
- RR A,B,C,D,E,10
- RR D,E,A,B,C,12
- RR B,C,D,E,A,14
- RR E,A,B,C,D,16
- RR C,D,E,A,B,18
-
- %xdefine F F2
-
- RR A,B,C,D,E,20
- RR D,E,A,B,C,22
- RR B,C,D,E,A,24
- RR E,A,B,C,D,26
- RR C,D,E,A,B,28
-
- RR A,B,C,D,E,30
- RR D,E,A,B,C,32
- RR B,C,D,E,A,34
- RR E,A,B,C,D,36
- RR C,D,E,A,B,38
-
- %xdefine F F3
-
- RR A,B,C,D,E,40
- RR D,E,A,B,C,42
- RR B,C,D,E,A,44
- RR E,A,B,C,D,46
- RR C,D,E,A,B,48
-
- RR A,B,C,D,E,50
- RR D,E,A,B,C,52
- RR B,C,D,E,A,54
- RR E,A,B,C,D,56
- RR C,D,E,A,B,58
-
- %xdefine F F4
-
- %if (%1 == 1)                         ;; if code loops through more than one block
- add   BUFFER_PTR, 64            ;; move to next 64-byte block
- cmp   BUFFER_PTR, BUFFER_END    ;; check if current block is the last one
- cmovae BUFFER_PTR, K_BASE       ;; smart way to signal the last iteration
- %else
- %xdefine W_NO_TAIL_PRECALC 1    ;; no software pipelining for single block interface
- %endif
-
- RR A,B,C,D,E,60
- RR D,E,A,B,C,62
- RR B,C,D,E,A,64
- RR E,A,B,C,D,66
- RR C,D,E,A,B,68
-
- RR A,B,C,D,E,70
- RR D,E,A,B,C,72
- RR B,C,D,E,A,74
- RR E,A,B,C,D,76
- RR C,D,E,A,B,78
-
- UPDATE_HASH [HASH_PTR   ],A
- UPDATE_HASH [HASH_PTR+ 4],B
- UPDATE_HASH [HASH_PTR+ 8],C
- UPDATE_HASH [HASH_PTR+12],D
- UPDATE_HASH [HASH_PTR+16],E
-
- %if (%1 == 1)
- jmp %%_loop
-
- align 32
- %%_end:
- %endif
-
-
- %xdefine W_NO_TAIL_PRECALC 0
- %xdefine F %error
-
-%endmacro
-
-
-%macro F1 3
- mov T1,%2
- xor T1,%3
- and T1,%1
- xor T1,%3
-%endmacro
-
-%macro F2 3
- mov T1,%3
- xor T1,%2
- xor T1,%1
-%endmacro
-
-%macro F3 3
- mov T1,%2
- mov T2,%1
- or  T1,%1
- and T2,%2
- and T1,%3
- or  T1,T2
-%endmacro
-
-%define F4 F2
-
-%macro UPDATE_HASH 2
- add %2, %1
- mov %1, %2
-%endmacro
-
-
-%macro W_PRECALC 1
- %xdefine i (%1)
-
- %if (i < 20)
- %xdefine K_XMM  0
- %elif (i < 40)
- %xdefine K_XMM  16
- %elif (i < 60)
- %xdefine K_XMM  32
- %else
- %xdefine K_XMM  48
- %endif
-
- %if (i<16 || (i>=80 && i<(80 + W_PRECALC_AHEAD)))
-
- %if (W_NO_TAIL_PRECALC == 0)
-
- %xdefine i ((%1) % 80)        ;; pre-compute for the next iteration
-
- %if (i == 0)
- W_PRECALC_RESET
- %endif
-
-
- W_PRECALC_00_15
- %endif
-
- %elif (i < 32)
- W_PRECALC_16_31
- %elif (i < 80)   ;; rounds 32-79
- W_PRECALC_32_79
- %endif
-%endmacro
-
-%macro W_PRECALC_RESET 0
- %xdefine    W             W0
- %xdefine    W_minus_04    W4
- %xdefine    W_minus_08    W8
- %xdefine    W_minus_12    W12
- %xdefine    W_minus_16    W16
- %xdefine    W_minus_20    W20
- %xdefine    W_minus_24    W24
- %xdefine    W_minus_28    W28
- %xdefine    W_minus_32    W
-%endmacro
-
-%macro W_PRECALC_ROTATE 0
- %xdefine    W_minus_32    W_minus_28
- %xdefine    W_minus_28    W_minus_24
- %xdefine    W_minus_24    W_minus_20
- %xdefine    W_minus_20    W_minus_16
- %xdefine    W_minus_16    W_minus_12
- %xdefine    W_minus_12    W_minus_08
- %xdefine    W_minus_08    W_minus_04
- %xdefine    W_minus_04    W
- %xdefine    W             W_minus_32
-%endmacro
-
-%xdefine W_PRECALC_AHEAD   16
-%xdefine W_NO_TAIL_PRECALC 0
-
-
-%xdefine xmm_mov            movdqa
-
-%macro W_PRECALC_00_15 0
- ;; message scheduling pre-compute for rounds 0-15
- %if ((i & 3) == 0)       ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
- movdqu W_TMP, [BUFFER_PTR + (i * 4)]
- %elif ((i & 3) == 1)
- pshufb W_TMP, XMM_SHUFB_BSWAP
- movdqa W, W_TMP
- %elif ((i & 3) == 2)
- paddd  W_TMP, [K_BASE]
- %elif ((i & 3) == 3)
- movdqa  [WK(i&~3)], W_TMP
-
- W_PRECALC_ROTATE
- %endif
-%endmacro
-
-%macro W_PRECALC_16_31 0
- ;; message scheduling pre-compute for rounds 16-31
- ;; calculating last 32 w[i] values in 8 XMM registers
- ;; pre-calculate K+w[i] values and store to mem, for later load by ALU add instruction
- ;;
- ;; "brute force" vectorization for rounds 16-31 only due to w[i]->w[i-3] dependency
- ;;
- %if ((i & 3) == 0)    ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
- movdqa  W, W_minus_12
- palignr W, W_minus_16, 8       ;; w[i-14]
- movdqa  W_TMP, W_minus_04
- psrldq  W_TMP, 4               ;; w[i-3]
- pxor    W, W_minus_08
- %elif ((i & 3) == 1)
- pxor    W_TMP, W_minus_16
- pxor    W, W_TMP
- movdqa  W_TMP2, W
- movdqa  W_TMP, W
- pslldq  W_TMP2, 12
- %elif ((i & 3) == 2)
- psrld   W, 31
- pslld   W_TMP, 1
- por     W_TMP, W
- movdqa  W, W_TMP2
- psrld   W_TMP2, 30
- pslld   W, 2
- %elif ((i & 3) == 3)
- pxor    W_TMP, W
- pxor    W_TMP, W_TMP2
- movdqa  W, W_TMP
- paddd   W_TMP, [K_BASE + K_XMM]
- movdqa  [WK(i&~3)],W_TMP
-
- W_PRECALC_ROTATE
- %endif
-%endmacro
-
-%macro W_PRECALC_32_79 0
- ;; in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
- ;; instead we do equal:    w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
- ;; allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
- ;;
- %if ((i & 3) == 0)    ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
- movdqa  W_TMP, W_minus_04
- pxor    W, W_minus_28         ;; W is W_minus_32 before xor
- palignr W_TMP, W_minus_08, 8
- %elif ((i & 3) == 1)
- pxor    W, W_minus_16
- pxor    W, W_TMP
- movdqa  W_TMP, W
- %elif ((i & 3) == 2)
- psrld   W, 30
- pslld   W_TMP, 2
- por     W_TMP, W
- %elif ((i & 3) == 3)
- movdqa  W, W_TMP
- paddd   W_TMP, [K_BASE + K_XMM]
- movdqa  [WK(i&~3)],W_TMP
-
- W_PRECALC_ROTATE
- %endif
-%endmacro
-
-%macro RR 6             ;; RR does two rounds of SHA-1 back to back with W pre-calculation
-
- ;;     TEMP = A
- ;;     A = F( i, B, C, D ) + E + ROTATE_LEFT( A, 5 ) + W[i] + K(i)
- ;;     C = ROTATE_LEFT( B, 30 )
- ;;     D = C
- ;;     E = D
- ;;     B = TEMP
-
- W_PRECALC (%6 + W_PRECALC_AHEAD)
- F    %2, %3, %4     ;; F returns result in T1
- add  %5, [WK(%6)]
- rol  %2, 30
- mov  T2, %1
- add  %4, [WK(%6 + 1)]
- rol  T2, 5
- add  %5, T1
-
- W_PRECALC (%6 + W_PRECALC_AHEAD + 1)
- add  T2, %5
- mov  %5, T2
- rol  T2, 5
- add  %4, T2
- F    %1, %2, %3    ;; F returns result in T1
- add  %4, T1
- rol  %1, 30
-
-;; write:  %1, %2
-;; rotate: %1<=%4, %2<=%5, %3<=%1, %4<=%2, %5<=%3
-%endmacro
-
-
-
-;;----------------------
-section .data align=128
-
-%xdefine K1 0x5a827999
-%xdefine K2 0x6ed9eba1
-%xdefine K3 0x8f1bbcdc
-%xdefine K4 0xca62c1d6
-
-align 128
-K_XMM_AR:
- DD K1, K1, K1, K1
- DD K2, K2, K2, K2
- DD K3, K3, K3, K3
- DD K4, K4, K4, K4
-
-align 16
-bswap_shufb_ctl:
- DD 00010203h
- DD 04050607h
- DD 08090a0bh
- DD 0c0d0e0fh
-
-;; dispatch pointer, points to the init routine for the first invocation
-sha1_update_intel_dispatched:
- DQ  sha1_update_intel_init_
-
-;;----------------------
-section .text align=4096
-
-SHA1_VECTOR_ASM     sha1_update_intel_ssse3_, multiblock
-
-align 32
-sha1_update_intel_init_:       ;; we get here with the first time invocation
- call    sha1_update_intel_dispacth_init_
-INTEL_SHA1_UPDATE_FUNCNAME:    ;; we get here after init
- jmp     qword [sha1_update_intel_dispatched]
-
-;; CPUID feature flag based dispatch
-sha1_update_intel_dispacth_init_:
- push    rax
- push    rbx
- push    rcx
- push    rdx
- push    rsi
-
- lea     rsi, [INTEL_SHA1_UPDATE_DEFAULT_DISPATCH]
-
- mov     eax, 1
- cpuid
-
- test    ecx, 0200h          ;; SSSE3 support, CPUID.1.ECX[bit 9]
- jz      _done
-
- lea     rsi, [sha1_update_intel_ssse3_]
-
-_done:
- mov     [sha1_update_intel_dispatched], rsi
-
- pop     rsi
- pop     rdx
- pop     rcx
- pop     rbx
- pop     rax
- ret
-
-;;----------------------
-;; in the case a default SHA-1 update function implementation was not provided
-;; and code was invoked on a non-SSSE3 supporting CPU, dispatch handles this
-;; failure in a safest way - jumps to the stub function with UD2 instruction below
-sha1_intel_non_ssse3_cpu_stub_:
- ud2     ;; in the case no default SHA-1 was provided non-SSSE3 CPUs safely fail here
- ret
-
-; END
-;----------------------