Add FreeBSD binary link and make a release

Previously scanMultipartDir() returns object info channel and timer
channel where timer channel is used to check whether object info
channel is alive or not.  This causes a race condition that timeout
may occur while object info channel in use.

This patch fixes the issue by removing timer channel and uses object
info channel directly where each object info has End bool field
indicates whether received object info is end or not.

.gitignore

@@ -10,7 +10,6 @@ site/
 /Minio.iml
 **/access.log
 build
-isa-l
 vendor/**/*.js
 vendor/**/*.json
 release

.travis.yml

@@ -6,16 +6,6 @@ os:
 - linux
 - osx
 
-before_install:
-  - git clone https://github.com/yasm/yasm
-  - cd yasm
-  - git checkout v1.3.0
-  - "./autogen.sh"
-  - "./configure"
-  - make
-  - export PATH=$PATH:`pwd`
-  - cd ..
-
 osx_image: xcode7.2
 
 env:

Bucket-Policy.md

@@ -9,14 +9,10 @@ This package implements parsing and validating bucket access policies based on A
 
 ### Supports following set of operations.
 
-    *
-    s3:*
     s3:GetObject
     s3:ListBucket
     s3:PutObject
-    s3:CreateBucket
     s3:GetBucketLocation
-    s3:DeleteBucket
     s3:DeleteObject
     s3:AbortMultipartUpload
     s3:ListBucketMultipartUploads
@@ -31,3 +27,7 @@ Supported applicable condition keys for each conditions.
 
     s3:prefix
     s3:max-keys
+
+### Nested policy support.
+
+Nested policies are not allowed.

Caddy.md

@@ -1,23 +1,27 @@
-### Proxy using Caddy.
+## Setting up Proxy using Caddy.
 
 Please download [Caddy Server](https://caddyserver.com/download)
 
 Create a caddy configuration file as below, change the ip addresses according to your local
 minio and DNS configuration.
 
+```bash
+$ ./minio --address localhost:9000 server <your_export_dir>
+```
+
 ```bash
 your.public.com {
-    proxy / 10.0.1.3:9000 {
+    proxy / localhost:9000 {
         proxy_header Host {host}
         proxy_header X-Real-IP {remote}
         proxy_header X-Forwarded-Proto {scheme}
     }
-    tls off
 }
 ```
 
 ```bash
 $ ./caddy
 Activating privacy features... done.
-your.public.com
+your.public.com:443
+your.public.com:80
 ```

Docker.md

@@ -1,38 +1,28 @@
-## Running Minio in Docker.
-
-### Installing Docker.
+### Run Minio docker image
 
+## Test Minio Docker Container
+Minio generates new access and secret keys each time you run this command. Container state is lost after you end this session. This mode is only intended for testing purpose.
 ```bash
-sudo apt-get install Docker.io
+docker run -p 9000:9000 minio/minio server /export
 ```
 
-### Generating `minio configs` for the first time.
+## Run Minio Docker Container
+Minio container requires a persistent volume to store configuration and application data. Following command maps local persistent directories from the host OS to virtual config `~/.minio` and export `/export` directories. 
 
 ```bash
-docker run -p 9000 minio/minio:latest
+docker run -p 9000:9000 --name minio1 \
+  -v /mnt/export/minio1:/export \
+  -v /mnt/config/minio1:/root/.minio \
+  minio/minio server /export
 ```
 
-### Persist `minio configs`.
-
+## Custom Access and Secret Keys
+To override Minio's auto-generated keys, you may pass secret and access keys explicitly as environment variables. Minio server also allows regular strings as access and secret keys.
 ```bash
-docker commit <running_minio_container_id> minio/my-minio
-docker stop <running_minio_container_id>
-```
-
-### Create a data volume container.
-
-```bash
-docker create -v /export --name minio-export minio/my-minio /bin/true
-```
-
-You can also map ``.minio`` directory containing authentication information.
-
-```bash
-docker create -v /export --name minio-export -v /.minio --name minio-config minio/my-minio /bin/true
-```
-
-You can then use the `--volumes-from` flag to mount the `/export` and ``/.minio`` volume in another container.
-
-```bash
-docker run -p 9000 --volumes-from minio-export --volumes-from minio-config --name minio-server1 minio/my-minio
+docker run -p 9000:9000 --name minio1 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -v /mnt/export/minio1:/export \
+  -v /mnt/config/minio1:/root/.minio \
+  minio/minio server /export
 ```

Makefile

@@ -19,10 +19,14 @@ endif
 ifeq ($(CPU), x86_64)
   HOST := $(HOST)64
 else
+ifeq ($(CPU), amd64)
+  HOST := $(HOST)64
+else
 ifeq ($(CPU), i686)
   HOST := $(HOST)32
 endif
 endif
+endif
 
 
 #############################################
@@ -41,6 +45,10 @@ ifndef (OS)
   else
   ifeq ($(HOST), Darwin32)
     arch = clang
+  else
+  ifeq ($(HOST), FreeBSD64)
+    arch = gcc
+  endif
   endif
   endif
   endif
@@ -60,12 +68,11 @@ checkdocker:
 
 getdeps: checks
 	@go get -u github.com/golang/lint/golint && echo "Installed golint:"
-	@go get -u golang.org/x/tools/cmd/vet && echo "Installed vet:"
 	@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
 	@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:"
 	@go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell:"
 
-verifiers: getdeps vet fmt lint cyclo spelling
+verifiers: vet fmt lint cyclo spelling
 
 vet:
 	@echo "Running $@:"
@@ -79,13 +86,6 @@ fmt:
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
 
-## Configure Intel library.
-isa-l:
-
-	@echo "Configuring $@:"
-	@git clone -q https://github.com/minio/isa-l.git
-	@cd isa-l; make -f Makefile.unx arch=$(arch) >/dev/null; mv include isa-l;
-
 lint:
 	@echo "Running $@:"
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
@@ -96,8 +96,7 @@ cyclo:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
 
-build: getdeps verifiers $(UI_ASSETS) isa-l
-	@GO15VENDOREXPERIMENT=1 go generate github.com/minio/minio/pkg/crypto/sha1
+build: getdeps verifiers $(UI_ASSETS)
 
 deadcode:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
@@ -108,12 +107,12 @@ spelling:
 
 test: build
 	@echo "Running all minio testing:"
-	@GODEBUG=cgocheck=0 CGO_CPPFLAGS="-I$(PWD)/isa-l" CGO_LDFLAGS="$(PWD)/isa-l/isa-l.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
-	@GODEBUG=cgocheck=0 CGO_CPPFLAGS="-I$(PWD)/isa-l" CGO_LDFLAGS="$(PWD)/isa-l/isa-l.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
 
 gomake-all: build
 	@echo "Installing minio:"
-	@CGO_CPPFLAGS="-I$(PWD)/isa-l" CGO_LDFLAGS="$(PWD)/isa-l/isa-l.a" GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
+	@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
 
 pkg-add:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
@@ -129,19 +128,18 @@ pkg-list:
 
 install: gomake-all
 
-dockerimage: checkdocker verifiers $(UI_ASSETS)
+dockerimage: checkdocker getdeps verifiers $(UI_ASSETS)
 	@echo "Building docker image:" minio:$(TAG)
-	@GO15VENDOREXPERIMENT=1 GOOS=linux GOARCH=amd64 go build --ldflags $(DOCKER_LDFLAGS) -o minio.dockerimage
-	@mkdir -p export
-	@sudo docker build --rm --tag=minio/minio:$(TAG) .
-	@rmdir export
-	@rm minio.dockerimage
+	@GO15VENDOREXPERIMENT=1 GOOS=linux GOARCH=amd64 go build --ldflags $(DOCKER_LDFLAGS) -o docker/minio.dockerimage
+	@cd docker; mkdir -p export; sudo docker build --rm --tag=minio/minio:$(TAG) .
+	@rmdir docker/export
+	@rm docker/minio.dockerimage
 
 release: verifiers
-	@MINIO_RELEASE=RELEASE GO15VENDOREXPERIMENT=1 ./buildscripts/build.sh
+	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
 
 experimental: verifiers
-	@MINIO_RELEASE=EXPERIMENTAL GO15VENDOREXPERIMENT=1 ./buildscripts/build.sh
+	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
 
 clean:
 	@echo "Cleaning up all the generated files:"

Multipart.md

@@ -0,0 +1,10 @@
+## Multipart backend format
+
+When multipart upload is used for objects, below meta-data/staging files are created 
+
+- New multipart upload call creates file ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.uploadid```
+- Put object part call creates file ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.PART_NUMBER.MD5SUM_STRING```
+- Abort multipart call removes all files matching ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.*```
+- Complete multipart call does
+  1. Create a staging file ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.complete.TEMP_NAME``` then rename to ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.complete```
+  2. Rename staging file ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.complete``` to ```EXPORT_DIR/BUCKET/PATH/TO/OBJECT```

README.md

@@ -2,11 +2,10 @@
 
 Minio is an object storage server compatible with Amazon S3 and licensed under [Apache license 2.0](./LICENSE).
 
-
 ## Description
 
 Minio is an open source object storage server released under Apache License V2. It is compatible with Amazon S3 cloud storage service. Minio follows a minimalist design philosophy.
-Minio is light enough to be bundled with the application stack. It sits on the side of NodeJS, Redis, MySQL and the likes. Unlike databases, Minio stores objects such as photos, videos, log files, backups, container / VM images and so on. Minio is best suited for storing blobs of information ranging from KBs to TBs each. In a simplistic sense, it is like a FTP server with a simple get / put API over HTTP.
+Minio is light enough to be bundled with the application stack. It sits on the side of NodeJS, Redis, MySQL and the likes. Unlike databases, Minio stores objects such as photos, videos, log files, backups, container / VM images and so on. Minio is best suited for storing blobs of information ranging from KBs to 5 TBs each. In a simplistic sense, it is like a FTP server with a simple get / put API over HTTP.
 
 Minio currently implements two backends
 
@@ -59,6 +58,15 @@ Download ``minio`` for:
 C:\Users\Username\Downloads> minio.exe --help
 ~~~
 
+#### FreeBSD
+
+Download ``minio`` from https://dl.minio.io/server/minio/release/freebsd-amd64/minio
+
+~~~
+$ chmod 755 minio
+$ ./minio --help
+~~~
+
 #### Docker container
 
 Download ``minio`` for docker.
@@ -84,46 +92,20 @@ $ make
 
 ### How to use Minio?
 
-```
-NAME:
-  minio server - Start Minio cloud storage server.
-
-USAGE:
-  minio server [OPTION VALUE] PATH
-
-  OPTION = expiry        VALUE = NN[h|m|s] [DEFAULT=Unlimited]
-  OPTION = min-free-disk VALUE = NN% [DEFAULT: 10%]
-
-EXAMPLES:
-  1. Start minio server on Linux.
-        $ minio server /home/shared
-
-  2. Start minio server on Windows.
-        $ minio server C:\MyShare
-
-  3. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
-        $ minio --address 192.168.1.101:9000 server /home/shared
-
-  4. Start minio server with minimum free disk threshold to 5%
-        $ minio server min-free-disk 5% /home/shared/Pictures
-
-  5. Start minio server with minimum free disk threshold to 15% with auto expiration set to 1h
-        $ minio server min-free-disk 15% expiry 1h /home/shared/Documents
-```
-
-#### Start Minio server.
+Start minio server.
 
 ~~~
 $ minio server ~/Photos
+
 AccessKey: WLGDGYAQYIGI833EV05A  SecretKey: BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF  Region: us-east-1
 
 Minio Object Storage:
     http://127.0.0.1:9000
-    http://10.0.0.3:9000
+    http://10.1.10.177:9000
 
 Minio Browser:
     http://127.0.0.1:9000
-    http://10.0.0.3:9000
+    http://10.1.10.177:9000
 
 To configure Minio Client:
     $ wget https://dl.minio.io/client/mc/release/darwin-amd64/mc
@@ -185,21 +167,64 @@ host_base = localhost:9000
 host_bucket = localhost:9000
 access_key = YOUR_ACCESS_KEY_HERE
 secret_key = YOUR_SECRET_KEY_HERE
+signature_v2 = False
 ```
 
-To list your buckets.
+To make a bucket
+```
+$ s3cmd mb s3://mybucket
+Bucket 's3://mybucket/' created
+```
+
+To copy an object to bucket
+```
+$ s3cmd put newfile.txt s3://testbucket
+upload: 'newfile' -> 's3://testbucket/newfile'  
+```
+
+To copy an object to local system
+```
+$ s3cmd get s3://testbucket/newfile
+download: 's3://testbucket/newfile' -> './newfile'
+```
+
+To sync local file/directory to a bucket 
+```
+$ s3cmd sync newdemo s3://testbucket
+upload: 'newdemo/newdemofile.txt' -> 's3://testbucket/newdemo/newdemofile.txt'
+```
+
+To sync bucket or object with local filesystem
+```
+$ s3cmd sync  s3://otherbucket otherlocalbucket
+download: 's3://otherbucket/cat.jpg' -> 'otherlocalbucket/cat.jpg' 
+```
+
+To list buckets.
 ```
 $ s3cmd ls s3://
 2015-12-09 16:12  s3://testbbucket
 ```
 
-To list contents inside buckets.
+To list contents inside bucket.
 ```
 $ s3cmd ls s3://testbucket/
                        DIR   s3://testbucket/test/
 2015-12-09 16:05    138504   s3://testbucket/newfile
 ```
 
+Delete an object from bucket
+```
+$ s3cmd del s3://testbucket/newfile
+delete: 's3://testbucket/newfile'
+```
+
+Delete a bucket
+```
+$ s3cmd rb s3://testbucket
+Bucket 's3://testbucket/' removed
+```
+
 ## Contribute to Minio Project
 Please follow Minio [Contributor's Guide](./CONTRIBUTING.md)
 

access-key.go

@@ -0,0 +1,96 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"regexp"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+// credential container for access and secret keys.
+type credential struct {
+	AccessKeyID     string `json:"accessKey"`
+	SecretAccessKey string `json:"secretKey"`
+}
+
+// stringer colorized access keys.
+func (a credential) String() string {
+	accessStr := colorMagenta("AccessKey: ") + colorWhite(a.AccessKeyID)
+	secretStr := colorMagenta("SecretKey: ") + colorWhite(a.SecretAccessKey)
+	return fmt.Sprint(accessStr + "  " + secretStr)
+}
+
+const (
+	minioAccessID = 20
+	minioSecretID = 40
+)
+
+// isValidSecretKey - validate secret key.
+var isValidSecretKey = regexp.MustCompile(`^.{8,40}$`)
+
+// isValidAccessKey - validate access key.
+var isValidAccessKey = regexp.MustCompile(`^[a-zA-Z0-9\\-\\.\\_\\~]{5,20}$`)
+
+// mustGenAccessKeys - must generate access credentials.
+func mustGenAccessKeys() (creds credential) {
+	creds, err := genAccessKeys()
+	fatalIf(err.Trace(), "Unable to generate access keys.", nil)
+	return creds
+}
+
+// genAccessKeys - generate access credentials.
+func genAccessKeys() (credential, *probe.Error) {
+	accessKeyID, err := genAccessKeyID()
+	if err != nil {
+		return credential{}, err.Trace()
+	}
+	secretAccessKey, err := genSecretAccessKey()
+	if err != nil {
+		return credential{}, err.Trace()
+	}
+	creds := credential{
+		AccessKeyID:     string(accessKeyID),
+		SecretAccessKey: string(secretAccessKey),
+	}
+	return creds, nil
+}
+
+// genAccessKeyID - generate random alpha numeric value using only uppercase characters
+// takes input as size in integer
+func genAccessKeyID() ([]byte, *probe.Error) {
+	alpha := make([]byte, minioAccessID)
+	if _, e := rand.Read(alpha); e != nil {
+		return nil, probe.NewError(e)
+	}
+	for i := 0; i < minioAccessID; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return alpha, nil
+}
+
+// genSecretAccessKey - generate random base64 numeric value from a random seed.
+func genSecretAccessKey() ([]byte, *probe.Error) {
+	rb := make([]byte, minioSecretID)
+	if _, e := rand.Read(rb); e != nil {
+		return nil, probe.NewError(e)
+	}
+	return []byte(base64.StdEncoding.EncodeToString(rb))[:minioSecretID], nil
+}

api-auth-utils.go

@@ -1,82 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"crypto/rand"
-	"encoding/base64"
-	"regexp"
-
-	"github.com/minio/minio/pkg/probe"
-)
-
-const (
-	minioAccessID = 20
-	minioSecretID = 40
-)
-
-// isValidAccessKey - validate access key
-func isValidAccessKey(accessKeyID string) bool {
-	if accessKeyID == "" {
-		return true
-	}
-	regex := regexp.MustCompile("^[A-Z0-9\\-\\.\\_\\~]{20}$")
-	return regex.MatchString(accessKeyID)
-}
-
-// isValidSecretKey - validate secret key
-func isValidSecretKey(secretKeyID string) bool {
-	regex := regexp.MustCompile("^[a-zA-Z0-9\\-\\.\\_\\~]{40}$")
-	return regex.MatchString(secretKeyID)
-}
-
-// generateAccessKeyID - generate random alpha numeric value using only uppercase characters
-// takes input as size in integer
-func generateAccessKeyID() ([]byte, *probe.Error) {
-	alpha := make([]byte, minioAccessID)
-	if _, e := rand.Read(alpha); e != nil {
-		return nil, probe.NewError(e)
-	}
-	for i := 0; i < minioAccessID; i++ {
-		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
-	}
-	return alpha, nil
-}
-
-// generateSecretAccessKey - generate random base64 numeric value from a random seed.
-func generateSecretAccessKey() ([]byte, *probe.Error) {
-	rb := make([]byte, minioSecretID)
-	if _, e := rand.Read(rb); e != nil {
-		return nil, probe.NewError(e)
-	}
-	return []byte(base64.StdEncoding.EncodeToString(rb))[:minioSecretID], nil
-}
-
-// mustGenerateAccessKeyID - must generate random alpha numeric value using only uppercase characters
-// takes input as size in integer
-func mustGenerateAccessKeyID() []byte {
-	alpha, err := generateAccessKeyID()
-	fatalIf(err.Trace(), "Unable to generate accessKeyID.", nil)
-	return alpha
-}
-
-// mustGenerateSecretAccessKey - generate random base64 numeric value from a random seed.
-func mustGenerateSecretAccessKey() []byte {
-	secretKey, err := generateSecretAccessKey()
-	fatalIf(err.Trace(), "Unable to generate secretAccessKey.", nil)
-	return secretKey
-}

api-errors.go

@@ -87,6 +87,23 @@ const (
 	ErrObjectExistsAsPrefix
 	ErrAllAccessDisabled
 	ErrMalformedPolicy
+	ErrMissingFields
+	ErrMissingCredTag
+	ErrCredMalformed
+	ErrInvalidRegion
+	ErrInvalidService
+	ErrInvalidRequestVersion
+	ErrMissingSignTag
+	ErrMissingSignHeadersTag
+	ErrPolicyAlreadyExpired
+	ErrMalformedDate
+	ErrMalformedExpires
+	ErrAuthHeaderEmpty
+	ErrDateHeaderMissing
+	ErrExpiredPresignRequest
+	ErrMissingDateHeader
+	ErrInvalidQuerySignatureAlgo
+	ErrInvalidQueryParams
 	// Add new error codes here.
 )
 
@@ -295,7 +312,87 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrMalformedPolicy: {
 		Code:           "MalformedPolicy",
-		Description:    "Policy has invalid resource",
+		Description:    "Policy has invalid resource.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingFields: {
+		Code:           "MissingFields",
+		Description:    "Missing fields in request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingCredTag: {
+		Code:           "InvalidRequest",
+		Description:    "Missing Credential field for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrCredMalformed: {
+		Code:           "CredentialMalformed",
+		Description:    "Credential field malformed does not follow accessKeyID/credScope.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedDate: {
+		Code:           "MalformedDate",
+		Description:    "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRegion: {
+		Code:           "InvalidRegion",
+		Description:    "Region does not match.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidService: {
+		Code:           "AccessDenied",
+		Description:    "Service scope should be of value 's3'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRequestVersion: {
+		Code:           "AccessDenied",
+		Description:    "Request scope should be of value 'aws4_request'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignTag: {
+		Code:           "AccessDenied",
+		Description:    "Signature header missing Signature field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignHeadersTag: {
+		Code:           "InvalidArgument",
+		Description:    "Signature header missing SignedHeaders field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPolicyAlreadyExpired: {
+		Code:           "AccessDenied",
+		Description:    "Invalid according to Policy: Policy expired.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedExpires: {
+		Code:           "MalformedExpires",
+		Description:    "Malformed expires header, expected non-zero number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthHeaderEmpty: {
+		Code:           "InvalidArgument",
+		Description:    "Authorization header is invalid -- one and only one ' ' (space) required.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingDateHeader: {
+		Code:           "AccessDenied",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQuerySignatureAlgo: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrExpiredPresignRequest: {
+		Code:           "AccessDenied",
+		Description:    "Request has expired.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQueryParams: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	// Add your error structure here.

api-headers.go

@@ -23,8 +23,6 @@ import (
 	"net/http"
 	"runtime"
 	"strconv"
-
-	"github.com/minio/minio/pkg/fs"
 )
 
 //// helpers
@@ -60,20 +58,20 @@ func encodeResponse(response interface{}) []byte {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, metadata fs.ObjectMetadata, contentRange *httpRange) {
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
 	// set common headers
 	setCommonHeaders(w)
 
 	// set object-related metadata headers
-	lastModified := metadata.LastModified.UTC().Format(http.TimeFormat)
+	lastModified := objInfo.ModifiedTime.UTC().Format(http.TimeFormat)
 	w.Header().Set("Last-Modified", lastModified)
 
-	w.Header().Set("Content-Type", metadata.ContentType)
-	if metadata.MD5 != "" {
-		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
+	w.Header().Set("Content-Type", objInfo.ContentType)
+	if objInfo.MD5Sum != "" {
+		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
 	}
 
-	w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
+	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
 
 	// for providing ranged content
 	if contentRange != nil {

api-resources.go

@@ -19,36 +19,52 @@ package main
 import (
 	"net/url"
 	"strconv"
-
-	"github.com/minio/minio/pkg/fs"
 )
 
-// parse bucket url queries
+// Parse bucket url queries
 func getBucketResources(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
 	prefix = values.Get("prefix")
 	marker = values.Get("marker")
 	delimiter = values.Get("delimiter")
-	maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
 	encodingType = values.Get("encoding-type")
 	return
 }
 
-// part bucket url queries for ?uploads
-func getBucketMultipartResources(values url.Values) (v fs.BucketMultipartResourcesMetadata) {
-	v.Prefix = values.Get("prefix")
-	v.KeyMarker = values.Get("key-marker")
-	v.MaxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
-	v.Delimiter = values.Get("delimiter")
-	v.EncodingType = values.Get("encoding-type")
-	v.UploadIDMarker = values.Get("upload-id-marker")
+// Parse bucket url queries for ?uploads
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+	prefix = values.Get("prefix")
+	keyMarker = values.Get("key-marker")
+	uploadIDMarker = values.Get("upload-id-marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-uploads") != "" {
+		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
+	} else {
+		maxUploads = maxUploadsList
+	}
+	encodingType = values.Get("encoding-type")
 	return
 }
 
-// parse object url queries
-func getObjectResources(values url.Values) (v fs.ObjectResourcesMetadata) {
-	v.UploadID = values.Get("uploadId")
-	v.PartNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
-	v.MaxParts, _ = strconv.Atoi(values.Get("max-parts"))
-	v.EncodingType = values.Get("encoding-type")
+// Parse object url queries
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
+	uploadID = values.Get("uploadId")
+	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+	if values.Get("max-parts") != "" {
+		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+	} else {
+		maxParts = maxPartsList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Get upload id.
+func getUploadID(values url.Values) (uploadID string) {
+	uploadID, _, _, _ = getObjectResources(values)
 	return
 }

api-response.go

@@ -20,15 +20,13 @@ import (
 	"encoding/xml"
 	"net/http"
 	"time"
-
-	"github.com/minio/minio/pkg/fs"
 )
 
 const (
-	// Reply date format
-	timeFormatAMZ = "2006-01-02T15:04:05.000Z"
-	// Limit number of objects in a given response.
-	maxObjectList = 1000
+	timeFormatAMZ  = "2006-01-02T15:04:05.000Z" // Reply date format
+	maxObjectList  = 1000                       // Limit number of objects in a listObjectsResponse.
+	maxUploadsList = 1000                       // Limit number of uploads in a listUploadsResponse.
+	maxPartsList   = 1000                       // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.
@@ -37,28 +35,6 @@ type LocationResponse struct {
 	Location string   `xml:",chardata"`
 }
 
-// AccessControlPolicyResponse - format for get bucket acl response.
-type AccessControlPolicyResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
-
-	AccessControlList struct {
-		Grants []Grant `xml:"Grant"`
-	}
-	Owner Owner
-}
-
-// Grant container for grantee and permission.
-type Grant struct {
-	Grantee struct {
-		ID           string
-		DisplayName  string
-		EmailAddress string
-		Type         string
-		URI          string
-	}
-	Permission string
-}
-
 // ListObjectsResponse - format for list objects response.
 type ListObjectsResponse struct {
 	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
@@ -247,7 +223,7 @@ func getLocation(r *http.Request) string {
 //
 // output:
 // populated struct that can be serialized to match xml and json api spec output
-func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsResponse {
+func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	var listbuckets []Bucket
 	var data = ListBucketsResponse{}
 	var owner = Owner{}
@@ -269,7 +245,7 @@ func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsRespons
 }
 
 // generates an ListObjects response for the said bucket with other enumerated options.
-func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp fs.ListObjectsResult) ListObjectsResponse {
+func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	var contents []Object
 	var prefixes []CommonPrefix
 	var owner = Owner{}
@@ -280,13 +256,13 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe
 
 	for _, object := range resp.Objects {
 		var content = Object{}
-		if object.Object == "" {
+		if object.Name == "" {
 			continue
 		}
-		content.Key = object.Object
-		content.LastModified = object.LastModified.UTC().Format(timeFormatAMZ)
-		if object.MD5 != "" {
-			content.ETag = "\"" + object.MD5 + "\""
+		content.Key = object.Name
+		content.LastModified = object.ModifiedTime.UTC().Format(timeFormatAMZ)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
 		}
 		content.Size = object.Size
 		content.StorageClass = "STANDARD"
@@ -341,56 +317,61 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string)
 }
 
 // generateListPartsResult
-func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPartsResponse {
+func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
 	// TODO - support EncodingType in xml decoding
 	listPartsResponse := ListPartsResponse{}
-	listPartsResponse.Bucket = objectMetadata.Bucket
-	listPartsResponse.Key = objectMetadata.Object
-	listPartsResponse.UploadID = objectMetadata.UploadID
+	listPartsResponse.Bucket = partsInfo.Bucket
+	listPartsResponse.Key = partsInfo.Object
+	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = "STANDARD"
 	listPartsResponse.Initiator.ID = "minio"
 	listPartsResponse.Initiator.DisplayName = "minio"
 	listPartsResponse.Owner.ID = "minio"
 	listPartsResponse.Owner.DisplayName = "minio"
 
-	listPartsResponse.MaxParts = objectMetadata.MaxParts
-	listPartsResponse.PartNumberMarker = objectMetadata.PartNumberMarker
-	listPartsResponse.IsTruncated = objectMetadata.IsTruncated
-	listPartsResponse.NextPartNumberMarker = objectMetadata.NextPartNumberMarker
+	listPartsResponse.MaxParts = partsInfo.MaxParts
+	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
+	listPartsResponse.IsTruncated = partsInfo.IsTruncated
+	listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
 
-	listPartsResponse.Parts = make([]Part, len(objectMetadata.Part))
-	for _, part := range objectMetadata.Part {
+	listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
+	for index, part := range partsInfo.Parts {
 		newPart := Part{}
 		newPart.PartNumber = part.PartNumber
 		newPart.ETag = "\"" + part.ETag + "\""
 		newPart.Size = part.Size
 		newPart.LastModified = part.LastModified.UTC().Format(timeFormatAMZ)
-		listPartsResponse.Parts = append(listPartsResponse.Parts, newPart)
+		listPartsResponse.Parts[index] = newPart
 	}
 	return listPartsResponse
 }
 
 // generateListMultipartUploadsResponse
-func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse {
+func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse {
 	listMultipartUploadsResponse := ListMultipartUploadsResponse{}
 	listMultipartUploadsResponse.Bucket = bucket
-	listMultipartUploadsResponse.Delimiter = metadata.Delimiter
-	listMultipartUploadsResponse.IsTruncated = metadata.IsTruncated
-	listMultipartUploadsResponse.EncodingType = metadata.EncodingType
-	listMultipartUploadsResponse.Prefix = metadata.Prefix
-	listMultipartUploadsResponse.KeyMarker = metadata.KeyMarker
-	listMultipartUploadsResponse.NextKeyMarker = metadata.NextKeyMarker
-	listMultipartUploadsResponse.MaxUploads = metadata.MaxUploads
-	listMultipartUploadsResponse.NextUploadIDMarker = metadata.NextUploadIDMarker
-	listMultipartUploadsResponse.UploadIDMarker = metadata.UploadIDMarker
-
-	listMultipartUploadsResponse.Uploads = make([]Upload, len(metadata.Upload))
-	for _, upload := range metadata.Upload {
+	listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter
+	listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated
+	listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType
+	listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix
+	listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker
+	listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker
+	listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads
+	listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker
+	listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker
+	listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes))
+	for index, commonPrefix := range multipartsInfo.CommonPrefixes {
+		listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{
+			Prefix: commonPrefix,
+		}
+	}
+	listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads))
+	for index, upload := range multipartsInfo.Uploads {
 		newUpload := Upload{}
 		newUpload.UploadID = upload.UploadID
 		newUpload.Key = upload.Object
-		newUpload.Initiated = upload.Initiated.Format(timeFormatAMZ)
-		listMultipartUploadsResponse.Uploads = append(listMultipartUploadsResponse.Uploads, newUpload)
+		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZ)
+		listMultipartUploadsResponse.Uploads[index] = newUpload
 	}
 	return listMultipartUploadsResponse
 }

api-router.go

@@ -0,0 +1,86 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import router "github.com/gorilla/mux"
+
+// objectStorageAPI container for S3 compatible API.
+type objectStorageAPI struct {
+	ObjectAPI ObjectAPI
+}
+
+// registerAPIRouter - registers S3 compatible APIs.
+func registerAPIRouter(mux *router.Router, api objectStorageAPI) {
+	// API Router
+	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
+
+	// Bucket router
+	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+
+	/// Object operations
+
+	// HeadObject
+	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
+	// PutObjectPart
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	// ListObjectPxarts
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
+	// CompleteMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// NewMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
+	// AbortMultipartUpload
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// GetObject
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
+	// CopyObject
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/).*?").HandlerFunc(api.CopyObjectHandler)
+	// PutObject
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
+	// DeleteObject
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
+
+	/// Bucket operations
+
+	// GetBucketLocation
+	bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
+	// GetBucketPolicy
+	bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
+	// ListMultipartUploads
+	bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
+	// ListObjects
+	bucket.Methods("GET").HandlerFunc(api.ListObjectsHandler)
+	// PutBucketPolicy
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
+	// PutBucket
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
+	// HeadBucket
+	bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
+	// PostPolicy
+	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
+	// DeleteMultipleObjects
+	bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
+	// DeleteBucketPolicy
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
+	// DeleteBucket
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
+
+	/// Root operation
+
+	// ListBuckets
+	apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
+}

appveyor.yml

@@ -10,36 +10,20 @@ environment:
   GOPATH: c:\gopath
   GO_EXTLINK_ENABLED: 0
   GO15VENDOREXPERIMENT: 1
-# Remove this after fixing erasure code.
-  GODEBUG: cgocheck=0
 
 # scripts that run after cloning repository
 install:
   - '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64'
-  - curl -fsSL -o c:\go\bin\yasm.exe http://www.tortall.net/projects/yasm/releases/yasm-1.3.0-win64.exe
   - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
   - rd C:\Go /s /q
   - appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.windows-amd64.zip
   - 7z x go1.6.windows-amd64.zip -oC:\ >nul
-  - curl -fsSL -o mingw-w64.7z http://downloads.sourceforge.net/project/mingw-w64-dgn/mingw-w64/mingw-w64-bin-x86_64-20151206.7z
-  - 7z x -oC:\ mingw-w64.7z > NUL
-  - set PATH=C:\mingw64\bin;%PATH%
-  - x86_64-w64-mingw32-gcc --version
   - go version
   - go env
-  - git clone https://github.com/minio/isa-l
-  - cd isa-l
-  - make -f Makefile.unx arch=mingw
-  - mv include isa-l
-  - set CGO_CPPFLAGS=-Ic:/gopath/src/github.com/minio/minio/isa-l
-  - set CGO_LDFLAGS=c:/gopath/src/github.com/minio/minio/isa-l/isa-l.a
-  - set CC=x86_64-w64-mingw32-gcc
-  - set CXX=x86_64-w64-mingw32-g++
   - cd %GOPATH%\src\github.com\minio\minio
 
 # to run your custom scripts instead of automatic MSBuild
 build_script:
-  - go generate github.com/minio/minio/pkg/crypto/sha1
   - go test .
   - go test -race .
   - go test github.com/minio/minio/pkg...

auth-handler.go

@@ -17,17 +17,16 @@
 package main
 
 import (
-	"crypto/sha256"
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
 	"encoding/hex"
+	"io/ioutil"
 	"net/http"
 	"strings"
 
-	"github.com/minio/minio/pkg/s3/signature4"
-)
-
-const (
-	signV4Algorithm = "AWS4-HMAC-SHA256"
-	jwtAlgorithm    = "Bearer"
+	fastSha256 "github.com/minio/minio/pkg/crypto/sha256"
+	"github.com/minio/minio/pkg/probe"
 )
 
 // Verify if request has JWT.
@@ -68,14 +67,6 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool {
 	return false
 }
 
-// Verify if incoming request is anonymous.
-func isRequestAnonymous(r *http.Request) bool {
-	if isRequestJWT(r) || isRequestSignatureV4(r) || isRequestPresignedSignatureV4(r) || isRequestPostPolicySignatureV4(r) {
-		return false
-	}
-	return true
-}
-
 // Authorization type.
 type authType int
 
@@ -105,30 +96,44 @@ func getRequestAuthType(r *http.Request) authType {
 	return authTypeUnknown
 }
 
+// sum256 calculate sha256 sum for an input byte array
+func sum256(data []byte) []byte {
+	hash := fastSha256.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// sumMD5 calculate md5 sum for an input byte array
+func sumMD5(data []byte) []byte {
+	hash := md5.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
 // Verify if request has valid AWS Signature Version '4'.
-func isReqAuthenticated(sign *signature4.Sign, r *http.Request) (s3Error APIErrorCode) {
-	auth := sign.SetHTTPRequestToVerify(r)
+func isReqAuthenticated(r *http.Request) (s3Error APIErrorCode) {
+	if r == nil {
+		errorIf(probe.NewError(errInvalidArgument), "HTTP request cannot be empty.", nil)
+		return ErrInternalError
+	}
+	payload, e := ioutil.ReadAll(r.Body)
+	if e != nil {
+		errorIf(probe.NewError(e), "Unable to read HTTP body.", nil)
+		return ErrInternalError
+	}
+	// Verify Content-Md5, if payload is set.
+	if r.Header.Get("Content-Md5") != "" {
+		if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+			return ErrBadDigest
+		}
+	}
+	// Populate back the payload.
+	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+	validateRegion := true // Validate region.
 	if isRequestSignatureV4(r) {
-		dummyPayload := sha256.Sum256([]byte(""))
-		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(dummyPayload[:]))
-		if err != nil {
-			errorIf(err.Trace(), "Signature verification failed.", nil)
-			return ErrInternalError
-		}
-		if !ok {
-			return ErrSignatureDoesNotMatch
-		}
-		return ErrNone
+		return doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
 	} else if isRequestPresignedSignatureV4(r) {
-		ok, err := auth.DoesPresignedSignatureMatch()
-		if err != nil {
-			errorIf(err.Trace(), "Presigned signature verification failed.", nil)
-			return ErrInternalError
-		}
-		if !ok {
-			return ErrSignatureDoesNotMatch
-		}
-		return ErrNone
+		return doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
 	}
 	return ErrAccessDenied
 }

bucket-handlers.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"bytes"
-	"crypto/md5"
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/xml"
@@ -30,23 +29,19 @@ import (
 	"strings"
 
 	mux "github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/crypto/sha256"
-	"github.com/minio/minio/pkg/fs"
 	"github.com/minio/minio/pkg/probe"
-	"github.com/minio/minio/pkg/s3/access"
-	"github.com/minio/minio/pkg/s3/signature4"
 )
 
-// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) {
 	// Read saved bucket policy.
 	policy, err := readBucketPolicy(bucket)
 	if err != nil {
 		errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
+		case BucketNotFound:
 			return ErrNoSuchBucket
-		case fs.BucketNameInvalid:
+		case BucketNameInvalid:
 			return ErrInvalidBucketName
 		default:
 			// For any other error just return AccessDenied.
@@ -54,14 +49,14 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error
 		}
 	}
 	// Parse the saved policy.
-	accessPolicy, e := accesspolicy.Validate(policy)
+	bucketPolicy, e := parseBucketPolicy(policy)
 	if e != nil {
 		errorIf(probe.NewError(e), "Parse policy failed.", nil)
 		return ErrAccessDenied
 	}
 
 	// Construct resource in 'arn:aws:s3:::examplebucket' format.
-	resource := accesspolicy.AWSResourcePrefix + strings.TrimPrefix(reqURL.Path, "/")
+	resource := AWSResourcePrefix + strings.TrimPrefix(reqURL.Path, "/")
 
 	// Get conditions for policy verification.
 	conditions := make(map[string]string)
@@ -70,7 +65,7 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error
 	}
 
 	// Validate action, resource and conditions with current policy statements.
-	if !bucketPolicyEvalStatements(action, resource, conditions, accessPolicy.Statements) {
+	if !bucketPolicyEvalStatements(action, resource, conditions, bucketPolicy.Statements) {
 		return ErrAccessDenied
 	}
 	return ErrNone
@@ -79,7 +74,7 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error
 // GetBucketLocationHandler - GET Bucket location.
 // -------------------------
 // This operation returns bucket location.
-func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -89,25 +84,46 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
 	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 		if s3Error := enforceBucketPolicy("s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	case authTypeSigned, authTypePresigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		payload, e := ioutil.ReadAll(r.Body)
+		if e != nil {
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		// Verify Content-Md5, if payload is set.
+		if r.Header.Get("Content-Md5") != "" {
+			if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+				writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
+				return
+			}
+		}
+		// Populate back the payload.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+		var s3Error APIErrorCode // API error code.
+		validateRegion := false  // Validate region.
+		if isRequestSignatureV4(r) {
+			s3Error = doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		} else if isRequestPresignedSignatureV4(r) {
+			s3Error = doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		}
+		if s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	_, err := api.Filesystem.GetBucketMetadata(bucket)
+	_, err := api.ObjectAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
+		errorIf(err.Trace(), "GetBucketInfo failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
+		case BucketNotFound:
 			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-		case fs.BucketNameInvalid:
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@@ -117,12 +133,14 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re
 
 	// Generate response.
 	encodedSuccessResponse := encodeResponse(LocationResponse{})
-	if api.Region != "us-east-1" {
+	// Get current region.
+	region := serverConfig.GetRegion()
+	if region != "us-east-1" {
 		encodedSuccessResponse = encodeResponse(LocationResponse{
-			Location: api.Region,
+			Location: region,
 		})
 	}
-	setCommonHeaders(w) // write headers.
+	setCommonHeaders(w) // Write headers.
 	writeSuccessResponse(w, encodedSuccessResponse)
 }
 
@@ -134,7 +152,7 @@ func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Re
 // completed or aborted. This operation returns at most 1,000 multipart
 // uploads in the response.
 //
-func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -150,26 +168,40 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http
 			return
 		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	resources := getBucketMultipartResources(r.URL.Query())
-	if resources.MaxUploads < 0 {
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
+	if maxUploads < 0 {
 		writeErrorResponse(w, r, ErrInvalidMaxUploads, r.URL.Path)
 		return
 	}
-	if resources.MaxUploads == 0 {
-		resources.MaxUploads = maxObjectList
+	if keyMarker != "" {
+		// Unescape keyMarker string
+		keyMarkerUnescaped, e := url.QueryUnescape(keyMarker)
+		if e != nil {
+			if e != nil {
+				// Return 'NoSuchKey' to indicate invalid marker key.
+				writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
+				return
+			}
+			keyMarker = keyMarkerUnescaped
+			// Marker not common with prefix is not implemented.
+			if !strings.HasPrefix(keyMarker, prefix) {
+				writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+				return
+			}
+		}
 	}
 
-	resources, err := api.Filesystem.ListMultipartUploads(bucket, resources)
+	listMultipartsInfo, err := api.ObjectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
 		errorIf(err.Trace(), "ListMultipartUploads failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
+		case BucketNotFound:
 			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@@ -177,7 +209,7 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http
 		return
 	}
 	// generate response
-	response := generateListMultipartUploadsResponse(bucket, resources)
+	response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo)
 	encodedSuccessResponse := encodeResponse(response)
 	// write headers.
 	setCommonHeaders(w)
@@ -191,7 +223,7 @@ func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
-func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -201,13 +233,13 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request)
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
 	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	case authTypeSigned, authTypePresigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
@@ -219,14 +251,32 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request)
 		writeErrorResponse(w, r, ErrInvalidMaxKeys, r.URL.Path)
 		return
 	}
-	if maxkeys == 0 {
-		maxkeys = maxObjectList
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+		return
+	}
+	// If marker is set unescape.
+	if marker != "" {
+		// Try to unescape marker.
+		markerUnescaped, e := url.QueryUnescape(marker)
+		if e != nil {
+			// Return 'NoSuchKey' to indicate invalid marker key.
+			writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
+			return
+		}
+		marker = markerUnescaped
+		// Marker not common with prefix is not implemented.
+		if !strings.HasPrefix(marker, prefix) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
 	}
 
-	listResp, err := api.Filesystem.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
+	listObjectsInfo, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
 	if err == nil {
 		// generate response
-		response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listResp)
+		response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listObjectsInfo)
 		encodedSuccessResponse := encodeResponse(response)
 		// Write headers
 		setCommonHeaders(w)
@@ -235,13 +285,13 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 	switch err.ToGoError().(type) {
-	case fs.BucketNameInvalid:
+	case BucketNameInvalid:
 		writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-	case fs.BucketNotFound:
+	case BucketNotFound:
 		writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-	case fs.ObjectNotFound:
+	case ObjectNotFound:
 		writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
-	case fs.ObjectNameInvalid:
+	case ObjectNameInvalid:
 		writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
 	default:
 		errorIf(err.Trace(), "ListObjects failed.", nil)
@@ -253,7 +303,7 @@ func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request)
 // -----------
 // This implementation of the GET operation returns a list of all buckets
 // owned by the authenticated sender of the request.
-func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	// List buckets does not support bucket policies.
 	switch getRequestAuthType(r) {
 	default:
@@ -261,16 +311,37 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request)
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
 	case authTypeSigned, authTypePresigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		payload, e := ioutil.ReadAll(r.Body)
+		if e != nil {
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		// Verify Content-Md5, if payload is set.
+		if r.Header.Get("Content-Md5") != "" {
+			if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+				writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
+				return
+			}
+		}
+		// Populate back the payload.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+		var s3Error APIErrorCode // API error code.
+		validateRegion := false  // Validate region.
+		if isRequestSignatureV4(r) {
+			s3Error = doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		} else if isRequestPresignedSignatureV4(r) {
+			s3Error = doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		}
+		if s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	buckets, err := api.Filesystem.ListBuckets()
+	bucketsInfo, err := api.ObjectAPI.ListBuckets()
 	if err == nil {
 		// generate response
-		response := generateListBucketsResponse(buckets)
+		response := generateListBucketsResponse(bucketsInfo)
 		encodedSuccessResponse := encodeResponse(response)
 		// write headers
 		setCommonHeaders(w)
@@ -283,10 +354,28 @@ func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request)
 }
 
 // DeleteMultipleObjectsHandler - deletes multiple objects.
-func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
@@ -301,9 +390,6 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Set http request for signature.
-	auth := api.Signature.SetHTTPRequestToVerify(r)
-
 	// Allocate incoming content length bytes.
 	deleteXMLBytes := make([]byte, r.ContentLength)
 
@@ -315,52 +401,6 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	switch getRequestAuthType(r) {
-	default:
-		// For all unknown auth types return error.
-		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
-		return
-	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
-			writeErrorResponse(w, r, s3Error, r.URL.Path)
-			return
-		}
-	case authTypePresigned:
-		// Check if request is presigned.
-		ok, err := auth.DoesPresignedSignatureMatch()
-		if err != nil {
-			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-	case authTypeSigned:
-		// Check if request is signed.
-		sha := sha256.New()
-		mdSh := md5.New()
-		sha.Write(deleteXMLBytes)
-		mdSh.Write(deleteXMLBytes)
-		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sha.Sum(nil)))
-		if err != nil {
-			errorIf(err.Trace(), "DeleteMultipleObjects failed.", nil)
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		// Verify content md5.
-		if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(mdSh.Sum(nil)) {
-			writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
-			return
-		}
-	}
-
 	// Unmarshal list of keys to be deleted.
 	deleteObjects := &DeleteObjectsRequest{}
 	if e := xml.Unmarshal(deleteXMLBytes, deleteObjects); e != nil {
@@ -372,7 +412,7 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
 	var deletedObjects []ObjectIdentifier
 	// Loop through all the objects and delete them sequentially.
 	for _, object := range deleteObjects.Objects {
-		err := api.Filesystem.DeleteObject(bucket, object.ObjectName)
+		err := api.ObjectAPI.DeleteObject(bucket, object.ObjectName)
 		if err == nil {
 			deletedObjects = append(deletedObjects, ObjectIdentifier{
 				ObjectName: object.ObjectName,
@@ -380,25 +420,25 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
 		} else {
 			errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil)
 			switch err.ToGoError().(type) {
-			case fs.BucketNameInvalid:
+			case BucketNameInvalid:
 				deleteErrors = append(deleteErrors, DeleteError{
 					Code:    errorCodeResponse[ErrInvalidBucketName].Code,
 					Message: errorCodeResponse[ErrInvalidBucketName].Description,
 					Key:     object.ObjectName,
 				})
-			case fs.BucketNotFound:
+			case BucketNotFound:
 				deleteErrors = append(deleteErrors, DeleteError{
 					Code:    errorCodeResponse[ErrNoSuchBucket].Code,
 					Message: errorCodeResponse[ErrNoSuchBucket].Description,
 					Key:     object.ObjectName,
 				})
-			case fs.ObjectNotFound:
+			case ObjectNotFound:
 				deleteErrors = append(deleteErrors, DeleteError{
 					Code:    errorCodeResponse[ErrNoSuchKey].Code,
 					Message: errorCodeResponse[ErrNoSuchKey].Description,
 					Key:     object.ObjectName,
 				})
-			case fs.ObjectNameInvalid:
+			case ObjectNameInvalid:
 				deleteErrors = append(deleteErrors, DeleteError{
 					Code:    errorCodeResponse[ErrNoSuchKey].Code,
 					Message: errorCodeResponse[ErrNoSuchKey].Description,
@@ -425,64 +465,31 @@ func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *htt
 // PutBucketHandler - PUT Bucket
 // ----------
 // This implementation of the PUT operation creates a new bucket for authenticated request
-func (api storageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	// Set http request for signature.
-	auth := api.Signature.SetHTTPRequestToVerify(r)
 	switch getRequestAuthType(r) {
 	default:
 		// For all unknown auth types return error.
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
-	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:CreateBucket", bucket, r.URL); s3Error != ErrNone {
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
-	case authTypePresigned:
-		ok, err := auth.DoesPresignedSignatureMatch()
-		if err != nil {
-			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-	case authTypeSigned:
-		// Verify signature for the incoming body if any.
-		locationBytes, e := ioutil.ReadAll(r.Body)
-		if e != nil {
-			errorIf(probe.NewError(e), "MakeBucket failed.", nil)
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-			return
-		}
-		sh := sha256.New()
-		sh.Write(locationBytes)
-		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
-		if err != nil {
-			errorIf(err.Trace(), "MakeBucket failed.", nil)
-			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
 	}
 
 	// Make bucket.
-	err := api.Filesystem.MakeBucket(bucket)
+	err := api.ObjectAPI.MakeBucket(bucket)
 	if err != nil {
 		errorIf(err.Trace(), "MakeBucket failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		case fs.BucketExists:
+		case BucketExists:
 			writeErrorResponse(w, r, ErrBucketAlreadyExists, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@@ -523,10 +530,9 @@ func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]stri
 // ----------
 // This implementation of the POST operation handles object creation with a specified
 // signature policy in multipart/form-data
-func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	// Here the parameter is the size of the form data that should
-	// be loaded in memory, the remaining being put in temporary
-	// files
+	// be loaded in memory, the remaining being put in temporary files.
 	reader, e := r.MultipartReader()
 	if e != nil {
 		errorIf(probe.NewError(e), "Unable to initialize multipart reader.", nil)
@@ -543,50 +549,38 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req
 	bucket := mux.Vars(r)["bucket"]
 	formValues["Bucket"] = bucket
 	object := formValues["Key"]
-	var ok bool
-
-	// Set http request for signature.
-	auth := api.Signature.SetHTTPRequestToVerify(r)
 
 	// Verify policy signature.
-	ok, err = auth.DoesPolicySignatureMatch(formValues)
+	apiErr := doesPolicySignatureMatch(formValues)
+	if apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+	if apiErr = checkPostPolicy(formValues); apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+	objInfo, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, nil)
 	if err != nil {
-		errorIf(err.Trace(), "Unable to verify signature.", nil)
-		writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-		return
-	}
-	if !ok {
-		writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-		return
-	}
-	if err = signature4.ApplyPolicyCond(formValues); err != nil {
-		errorIf(err.Trace(), "Invalid request, policy doesn't match with the endpoint.", nil)
-		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
-		return
-	}
-	metadata, err := api.Filesystem.CreateObject(bucket, object, "", -1, fileBody, nil)
-	if err != nil {
-		errorIf(err.Trace(), "CreateObject failed.", nil)
+		errorIf(err.Trace(), "PutObject failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.RootPathFull:
+		case RootPathFull:
 			writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
-		case fs.BucketNotFound:
+		case BucketNotFound:
 			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-		case fs.BucketNameInvalid:
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		case fs.BadDigest:
+		case BadDigest:
 			writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
-		case fs.IncompleteBody:
+		case IncompleteBody:
 			writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path)
-		case fs.InvalidDigest:
-			writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		}
 		return
 	}
-	if metadata.MD5 != "" {
-		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
+	if objInfo.MD5Sum != "" {
+		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
 	}
 	writeSuccessResponse(w, nil)
 }
@@ -597,7 +591,7 @@ func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Req
 // The operation returns a 200 OK if the bucket exists and you
 // have permission to access it. Otherwise, the operation might
 // return responses such as 404 Not Found and 403 Forbidden.
-func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -606,20 +600,26 @@ func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request)
 		// For all unknown auth types return error.
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	_, err := api.Filesystem.GetBucketMetadata(bucket)
+	_, err := api.ObjectAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
+		errorIf(err.Trace(), "GetBucketInfo failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
+		case BucketNotFound:
 			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-		case fs.BucketNameInvalid:
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@@ -630,7 +630,7 @@ func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request)
 }
 
 // DeleteBucketHandler - Delete bucket
-func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -639,26 +639,20 @@ func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request
 		// For all unknown auth types return error.
 		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
-	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy("s3:DeleteBucket", bucket, r.URL); s3Error != ErrNone {
-			writeErrorResponse(w, r, s3Error, r.URL.Path)
-			return
-		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	err := api.Filesystem.DeleteBucket(bucket)
+	err := api.ObjectAPI.DeleteBucket(bucket)
 	if err != nil {
 		errorIf(err.Trace(), "DeleteBucket failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
+		case BucketNotFound:
 			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
-		case fs.BucketNotEmpty:
+		case BucketNotEmpty:
 			writeErrorResponse(w, r, ErrBucketNotEmpty, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)

bucket-policy-handlers.go

@@ -18,8 +18,6 @@ package main
 
 import (
 	"bytes"
-	"crypto/sha256"
-	"encoding/hex"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -27,9 +25,7 @@ import (
 	"strings"
 
 	mux "github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/fs"
 	"github.com/minio/minio/pkg/probe"
-	"github.com/minio/minio/pkg/s3/access"
 )
 
 // maximum supported access policy size.
@@ -37,12 +33,13 @@ const maxAccessPolicySize = 20 * 1024 * 1024 // 20KiB.
 
 // Verify if a given action is valid for the url path based on the
 // existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions map[string]string, statements []accesspolicy.Statement) bool {
+func bucketPolicyEvalStatements(action string, resource string, conditions map[string]string, statements []policyStatement) bool {
 	for _, statement := range statements {
 		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
 			if statement.Effect == "Allow" {
 				return true
 			}
+			// Do not uncomment kept here for readability.
 			// else statement.Effect == "Deny"
 			return false
 		}
@@ -52,7 +49,7 @@ func bucketPolicyEvalStatements(action string, resource string, conditions map[s
 }
 
 // Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions map[string]string, statement accesspolicy.Statement) bool {
+func bucketPolicyMatchStatement(action string, resource string, conditions map[string]string, statement policyStatement) bool {
 	// Verify if action matches.
 	if bucketPolicyActionMatch(action, statement) {
 		// Verify if resource matches.
@@ -67,7 +64,7 @@ func bucketPolicyMatchStatement(action string, resource string, conditions map[s
 }
 
 // Verify if given action matches with policy statement.
-func bucketPolicyActionMatch(action string, statement accesspolicy.Statement) bool {
+func bucketPolicyActionMatch(action string, statement policyStatement) bool {
 	for _, policyAction := range statement.Actions {
 		// Policy action can be a regex, validate the action with matching string.
 		matched, e := regexp.MatchString(policyAction, action)
@@ -80,7 +77,7 @@ func bucketPolicyActionMatch(action string, statement accesspolicy.Statement) bo
 }
 
 // Verify if given resource matches with policy statement.
-func bucketPolicyResourceMatch(resource string, statement accesspolicy.Statement) bool {
+func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
 	for _, presource := range statement.Resources {
 		matched, e := regexp.MatchString(presource, strings.TrimPrefix(resource, "/"))
 		fatalIf(probe.NewError(e), "Invalid pattern, please verify the pattern string.", nil)
@@ -93,7 +90,7 @@ func bucketPolicyResourceMatch(resource string, statement accesspolicy.Statement
 }
 
 // Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions map[string]string, statement accesspolicy.Statement) bool {
+func bucketPolicyConditionMatch(conditions map[string]string, statement policyStatement) bool {
 	// Supports following conditions.
 	// - StringEquals
 	// - StringNotEquals
@@ -130,10 +127,22 @@ func bucketPolicyConditionMatch(conditions map[string]string, statement accesspo
 // -----------------
 // This implementation of the PUT operation uses the policy
 // subresource to add to or replace a policy on a bucket
-func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
 	// If Content-Length is unknown or zero, deny the
 	// request. PutBucketPolicy always needs a Content-Length if
 	// incoming request is not chunked.
@@ -152,65 +161,33 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ
 	// Read access policy up to maxAccessPolicySize.
 	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 	// bucket policies are limited to 20KB in size, using a limit reader.
-	accessPolicyBytes, e := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	bucketPolicyBuf, e := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
 	if e != nil {
 		errorIf(probe.NewError(e).Trace(bucket), "Reading policy failed.", nil)
 		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		return
 	}
 
-	// Parse access access.
-	accessPolicy, e := accesspolicy.Validate(accessPolicyBytes)
+	// Parse bucket policy.
+	bucketPolicy, e := parseBucketPolicy(bucketPolicyBuf)
 	if e != nil {
+		errorIf(probe.NewError(e), "Unable to parse bucket policy.", nil)
 		writeErrorResponse(w, r, ErrInvalidPolicyDocument, r.URL.Path)
 		return
 	}
 
-	// If the policy resource has different bucket name, reject it.
-	for _, statement := range accessPolicy.Statements {
-		for _, resource := range statement.Resources {
-			resourcePrefix := strings.SplitAfter(resource, accesspolicy.AWSResourcePrefix)[1]
-			if !strings.HasPrefix(resourcePrefix, bucket) {
-				writeErrorResponse(w, r, ErrMalformedPolicy, r.URL.Path)
-				return
-			}
-		}
-	}
-
-	// Set http request for signature verification.
-	auth := api.Signature.SetHTTPRequestToVerify(r)
-	if isRequestPresignedSignatureV4(r) {
-		ok, err := auth.DoesPresignedSignatureMatch()
-		if err != nil {
-			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-	} else if isRequestSignatureV4(r) {
-		sh := sha256.New()
-		sh.Write(accessPolicyBytes)
-		ok, err := api.Signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
-		if err != nil {
-			errorIf(err.Trace(string(accessPolicyBytes)), "SaveBucketPolicy failed.", nil)
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
-		if !ok {
-			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
-			return
-		}
+	// Parse check bucket policy.
+	if s3Error := checkBucketPolicy(bucket, bucketPolicy); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
 	}
 
 	// Save bucket policy.
-	err := writeBucketPolicy(bucket, accessPolicyBytes)
+	err := writeBucketPolicy(bucket, bucketPolicyBuf)
 	if err != nil {
-		errorIf(err.Trace(bucket), "SaveBucketPolicy failed.", nil)
+		errorIf(err.Trace(bucket, string(bucketPolicyBuf)), "SaveBucketPolicy failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@@ -224,14 +201,20 @@ func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Requ
 // -----------------
 // This implementation of the DELETE operation uses the policy
 // subresource to add to remove a policy on a bucket.
-func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	// Validate incoming signature.
-	if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
-		writeErrorResponse(w, r, s3Error, r.URL.Path)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
 	}
 
 	// Delete bucket access policy.
@@ -239,9 +222,9 @@ func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.R
 	if err != nil {
 		errorIf(err.Trace(bucket), "DeleteBucketPolicy failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		case fs.BucketPolicyNotFound:
+		case BucketPolicyNotFound:
 			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
@@ -255,14 +238,20 @@ func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.R
 // -----------------
 // This operation uses the policy
 // subresource to return the policy of a specified bucket.
-func (api storageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectStorageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	// Validate incoming signature.
-	if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
-		writeErrorResponse(w, r, s3Error, r.URL.Path)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
 	}
 
 	// Read bucket access policy.
@@ -270,9 +259,9 @@ func (api storageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Requ
 	if err != nil {
 		errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
 		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
+		case BucketNameInvalid:
 			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
-		case fs.BucketPolicyNotFound:
+		case BucketPolicyNotFound:
 			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
 		default:
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)

bucket-policy-parser.go

@@ -14,14 +14,16 @@
  * limitations under the License.
  */
 
-// Package accesspolicy implements AWS Access Policy Language parser in
+// This file implements AWS Access Policy Language parser in
 // accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-package accesspolicy
+package main
 
 import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"regexp"
+	"sort"
 	"strings"
 )
 
@@ -32,14 +34,10 @@ const (
 
 // supportedActionMap - lists all the actions supported by minio.
 var supportedActionMap = map[string]struct{}{
-	"*":                             {},
-	"s3:*":                          {},
 	"s3:GetObject":                  {},
 	"s3:ListBucket":                 {},
 	"s3:PutObject":                  {},
-	"s3:CreateBucket":               {},
 	"s3:GetBucketLocation":          {},
-	"s3:DeleteBucket":               {},
 	"s3:DeleteObject":               {},
 	"s3:AbortMultipartUpload":       {},
 	"s3:ListBucketMultipartUploads": {},
@@ -47,15 +45,15 @@ var supportedActionMap = map[string]struct{}{
 }
 
 // User - canonical users list.
-type User struct {
+type policyUser struct {
 	AWS []string
 }
 
 // Statement - minio policy statement
-type Statement struct {
+type policyStatement struct {
 	Sid        string
 	Effect     string
-	Principal  User
+	Principal  policyUser                   `json:"Principal"`
 	Actions    []string                     `json:"Action"`
 	Resources  []string                     `json:"Resource"`
 	Conditions map[string]map[string]string `json:"Condition"`
@@ -63,8 +61,8 @@ type Statement struct {
 
 // BucketPolicy - minio policy collection
 type BucketPolicy struct {
-	Version    string      // date in 0000-00-00 format
-	Statements []Statement `json:"Statement"`
+	Version    string            // date in 0000-00-00 format
+	Statements []policyStatement `json:"Statement"`
 }
 
 // supportedEffectMap - supported effects.
@@ -183,9 +181,71 @@ func isValidConditions(conditions map[string]map[string]string) (err error) {
 	return nil
 }
 
-// Validate - validate if request body is of proper JSON and in
-// accordance with policy standards.
-func Validate(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
+// List of actions for which prefixes are not allowed.
+var invalidPrefixActions = map[string]struct{}{
+	"s3:GetBucketLocation":          {},
+	"s3:ListBucket":                 {},
+	"s3:ListBucketMultipartUploads": {},
+	// Add actions which do not honor prefixes.
+}
+
+// checkBucketPolicy validates unmarshalled bucket policy structure.
+func checkBucketPolicy(bucket string, bucketPolicy BucketPolicy) APIErrorCode {
+	// Validate statements for special actions and collect resources
+	// for others to validate nesting.
+	var resourceMap = make(map[string]struct{})
+	for _, statement := range bucketPolicy.Statements {
+		for _, action := range statement.Actions {
+			for _, resource := range statement.Resources {
+				resourcePrefix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+				if _, ok := invalidPrefixActions[action]; ok {
+					// Resource prefix is not equal to bucket for
+					// prefix invalid actions, reject them.
+					if resourcePrefix != bucket {
+						return ErrMalformedPolicy
+					}
+				} else {
+					// For all other actions validate if prefix begins
+					// with bucket, if not reject them.
+					if !strings.HasPrefix(resourcePrefix, bucket) {
+						return ErrMalformedPolicy
+					}
+					// All valid resources collect them separately to verify nesting.
+					resourceMap[resourcePrefix] = struct{}{}
+				}
+			}
+		}
+	}
+
+	var resources []string
+	for resource := range resourceMap {
+		resources = append(resources, resource)
+	}
+
+	// Sort strings as shorter first.
+	sort.Strings(resources)
+
+	for len(resources) > 1 {
+		var resource string
+		resource, resources = resources[0], resources[1:]
+		resourceRegex := regexp.MustCompile(resource)
+		// Loop through all resources, if one of them matches with
+		// previous shorter one, it means we have detected
+		// nesting. Reject such rules.
+		for _, otherResource := range resources {
+			if resourceRegex.MatchString(otherResource) {
+				return ErrMalformedPolicy
+			}
+		}
+	}
+
+	// No errors found.
+	return ErrNone
+}
+
+// parseBucketPolicy - parses and validates if bucket policy is of
+// proper JSON and follows allowed restrictions with policy standards.
+func parseBucketPolicy(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
 	if err = json.Unmarshal(bucketPolicyBuf, &policy); err != nil {
 		return BucketPolicy{}, err
 	}
@@ -216,7 +276,7 @@ func Validate(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
 		if err := isValidActions(statement.Actions); err != nil {
 			return BucketPolicy{}, err
 		}
-		// Statment resources should be valid.
+		// Statement resources should be valid.
 		if err := isValidResources(statement.Resources); err != nil {
 			return BucketPolicy{}, err
 		}
@@ -225,6 +285,23 @@ func Validate(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
 			return BucketPolicy{}, err
 		}
 	}
+
+	// Separate deny and allow statements, so that we can apply deny
+	// statements in the beginning followed by Allow statements.
+	var denyStatements []policyStatement
+	var allowStatements []policyStatement
+	for _, statement := range policy.Statements {
+		if statement.Effect == "Deny" {
+			denyStatements = append(denyStatements, statement)
+			continue
+		}
+		// else if statement.Effect == "Allow"
+		allowStatements = append(allowStatements, statement)
+	}
+
+	// Deny statements are enforced first once matched.
+	policy.Statements = append(denyStatements, allowStatements...)
+
 	// Return successfully parsed policy structure.
 	return policy, nil
 }

bucket-policy.go

@@ -21,7 +21,6 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/minio/minio/pkg/fs"
 	"github.com/minio/minio/pkg/probe"
 )
 
@@ -46,7 +45,7 @@ func createBucketsConfigPath() *probe.Error {
 	return nil
 }
 
-// getBucketConfigPath - get bucket path.
+// getBucketConfigPath - get bucket config path.
 func getBucketConfigPath(bucket string) (string, *probe.Error) {
 	bucketsConfigPath, err := getBucketsConfigPath()
 	if err != nil {
@@ -55,7 +54,7 @@ func getBucketConfigPath(bucket string) (string, *probe.Error) {
 	return filepath.Join(bucketsConfigPath, bucket), nil
 }
 
-// createBucketConfigPath - create bucket directory.
+// createBucketConfigPath - create bucket config directory.
 func createBucketConfigPath(bucket string) *probe.Error {
 	bucketConfigPath, err := getBucketConfigPath(bucket)
 	if err != nil {
@@ -70,8 +69,8 @@ func createBucketConfigPath(bucket string) *probe.Error {
 // readBucketPolicy - read bucket policy.
 func readBucketPolicy(bucket string) ([]byte, *probe.Error) {
 	// Verify bucket is valid.
-	if !fs.IsValidBucketName(bucket) {
-		return nil, probe.NewError(fs.BucketNameInvalid{Bucket: bucket})
+	if !IsValidBucketName(bucket) {
+		return nil, probe.NewError(BucketNameInvalid{Bucket: bucket})
 	}
 
 	bucketConfigPath, err := getBucketConfigPath(bucket)
@@ -83,7 +82,7 @@ func readBucketPolicy(bucket string) ([]byte, *probe.Error) {
 	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
 	if _, e := os.Stat(bucketPolicyFile); e != nil {
 		if os.IsNotExist(e) {
-			return nil, probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket})
+			return nil, probe.NewError(BucketPolicyNotFound{Bucket: bucket})
 		}
 		return nil, probe.NewError(e)
 	}
@@ -98,8 +97,8 @@ func readBucketPolicy(bucket string) ([]byte, *probe.Error) {
 // removeBucketPolicy - remove bucket policy.
 func removeBucketPolicy(bucket string) *probe.Error {
 	// Verify bucket is valid.
-	if !fs.IsValidBucketName(bucket) {
-		return probe.NewError(fs.BucketNameInvalid{Bucket: bucket})
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
 	}
 
 	bucketConfigPath, err := getBucketConfigPath(bucket)
@@ -111,7 +110,7 @@ func removeBucketPolicy(bucket string) *probe.Error {
 	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
 	if _, e := os.Stat(bucketPolicyFile); e != nil {
 		if os.IsNotExist(e) {
-			return probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket})
+			return probe.NewError(BucketPolicyNotFound{Bucket: bucket})
 		}
 		return probe.NewError(e)
 	}
@@ -121,8 +120,13 @@ func removeBucketPolicy(bucket string) *probe.Error {
 // writeBucketPolicy - save bucket policy.
 func writeBucketPolicy(bucket string, accessPolicyBytes []byte) *probe.Error {
 	// Verify if bucket path legal
-	if !fs.IsValidBucketName(bucket) {
-		return probe.NewError(fs.BucketNameInvalid{Bucket: bucket})
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	// Create bucket config path.
+	if err := createBucketConfigPath(bucket); err != nil {
+		return err.Trace()
 	}
 
 	bucketConfigPath, err := getBucketConfigPath(bucket)

buildscripts/build.sh

@@ -23,7 +23,12 @@ _init() {
     fi
 
     # List of supported architectures
-    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm windows/386 windows/amd64 darwin/amd64'
+    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm windows/386 windows/amd64 darwin/amd64 freebsd/amd64'
+
+    ## System binaries
+    CP=`which cp`
+    SHASUM=`which shasum`
+    SED=`which sed`
 }
 
 go_build() {
@@ -33,15 +38,46 @@ go_build() {
     package=$(go list -f '{{.ImportPath}}')
     echo -n "-->"
     printf "%15s:%s\n" "${osarch}" "${package}"
-    GO15VENDOREXPERIMENT=1 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_str/$os-$arch/$(basename $package).$release_tag
+
+    # Release binary name
+    release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
+    # Release binary downloadable name
+    release_real_bin="$release_str/$os-$arch/$(basename $package)"
+    # Release shasum name
+    release_shasum="$release_str/$os-$arch/$(basename $package).shasum"
+
+    # Go build to build the binary.
+    GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
+
+    # Create copy
+    if [ $os == "windows" ]; then
+        $CP -p $release_bin ${release_real_bin}.exe
+    else
+        $CP -p $release_bin $release_real_bin
+    fi
+
+    # Calculate shasum
+    shasum_str=$(${SHASUM} ${release_bin})
+    echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
 }
 
 main() {
     # Build releases.
     echo "Executing $release_str builds for OS: ${SUPPORTED_OSARCH}"
+    echo  "Choose an OS Arch from the below"
     for osarch in ${SUPPORTED_OSARCH}; do
-        go_build ${osarch}
+        echo ${osarch}
     done
+
+    read -p "If you want to build for all, Just press Enter: " chosen_osarch
+    if [ "$chosen_osarch" = "" ]; then
+        for each_osarch in ${SUPPORTED_OSARCH}; do
+            go_build ${each_osarch}
+        done
+    else
+        go_build ${chosen_osarch}
+    fi
+
 }
 
 # Run main.

buildscripts/checkdeps.sh

@@ -21,7 +21,6 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    YASM_VERSION="1.2.0"
     GO_VERSION="1.6"
     OSX_VERSION="10.8"
     UNAME=$(uname -sm)
@@ -141,6 +140,9 @@ is_supported_os() {
         "Linux")
             os="linux"
             ;;
+        "FreeBSD")
+            os="freebsd"
+            ;;
         "Darwin")
             osx_host_version=$(env sw_vers -productVersion)
             check_version "${osx_host_version}" "${OSX_VERSION}"
@@ -155,7 +157,7 @@ is_supported_os() {
 is_supported_arch() {
     local supported
     case ${UNAME##* } in
-        "x86_64")
+        "x86_64" | "amd64")
             supported=1
             ;;
         "arm"*)
@@ -181,11 +183,6 @@ check_deps() {
     if [ $? -ge 2 ]; then
         MISSING="${MISSING} git"
     fi
-
-    check_version "$(env yasm --version 2>/dev/null | sed 's/^.* \([0-9.]*\).*$/\1/' | head -1)" "${YASM_VERSION}"
-    if [ $? -ge 2 ]; then
-        MISSING="${MISSING} yasm(${YASM_VERSION})"
-    fi
 }
 
 main() {

buildscripts/checkgopath.sh

@@ -19,8 +19,8 @@ _init() {
 
     shopt -s extglob
 
-    PWD=$(pwd)
-    GOPATH=$(go env GOPATH)
+    PWD=$(pwd -P)
+    GOPATH=$(cd "$(go env GOPATH)" ; env pwd -P)
 }
 
 main() {

certs.go

@@ -0,0 +1,94 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/go-homedir"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// createCertsPath create certs path.
+func createCertsPath() *probe.Error {
+	certsPath, err := getCertsPath()
+	if err != nil {
+		return err.Trace()
+	}
+	if err := os.MkdirAll(certsPath, 0700); err != nil {
+		return probe.NewError(err)
+	}
+	return nil
+}
+
+// getCertsPath get certs path.
+func getCertsPath() (string, *probe.Error) {
+	homeDir, e := homedir.Dir()
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+	certsPath := filepath.Join(homeDir, globalMinioCertsDir)
+	return certsPath, nil
+}
+
+// mustGetCertsPath must get certs path.
+func mustGetCertsPath() string {
+	certsPath, err := getCertsPath()
+	fatalIf(err.Trace(), "Unable to retrieve certs path.", nil)
+	return certsPath
+}
+
+// mustGetCertFile must get cert file.
+func mustGetCertFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioCertFile)
+}
+
+// mustGetKeyFile must get key file.
+func mustGetKeyFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioKeyFile)
+}
+
+// isCertFileExists verifies if cert file exists, returns true if
+// found, false otherwise.
+func isCertFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioCertFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isKeyFileExists verifies if key file exists, returns true if found,
+// false otherwise.
+func isKeyFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioKeyFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isSSL - returns true with both cert and key exists.
+func isSSL() bool {
+	if isCertFileExists() && isKeyFileExists() {
+		return true
+	}
+	return false
+}

commands.go

@@ -18,13 +18,30 @@ package main
 
 import "github.com/minio/cli"
 
-// Collection of minio commands currently supported are
+// Collection of minio commands currently supported are.
 var commands = []cli.Command{}
 
-// Collection of minio commands currently supported in a trie tree
+// Collection of minio commands currently supported in a trie tree.
 var commandsTree = newTrie()
 
-// registerCommand registers a cli command
+// Collection of minio flags currently supported.
+var globalFlags = []cli.Flag{
+	cli.BoolFlag{
+		Name:  "quiet, q",
+		Usage: "Suppress chatty console output.",
+	},
+	cli.BoolFlag{
+		Name:  "debug",
+		Usage: "Enable debugging output.",
+	},
+	cli.StringFlag{
+		Name:  "config-dir, C",
+		Value: mustGetConfigPath(),
+		Usage: "Path to configuration folder.",
+	},
+}
+
+// registerCommand registers a cli command.
 func registerCommand(command cli.Command) {
 	commands = append(commands, command)
 	commandsTree.Insert(command.Name)

config-logger-main.go

@@ -1,164 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"runtime"
-
-	"github.com/minio/cli"
-	"github.com/minio/mc/pkg/console"
-	"github.com/minio/minio/pkg/probe"
-)
-
-// Configure logger
-var configLoggerCmd = cli.Command{
-	Name:   "logger",
-	Usage:  "Configure logger.",
-	Action: mainConfigLogger,
-	CustomHelpTemplate: `NAME:
-   minio config {{.Name}} - {{.Usage}}
-
-USAGE:
-   minio config {{.Name}} OPERATION [ARGS...]
-
-   OPERATION = add | list | remove
-
-EXAMPLES:
-   1. Configure new mongo logger.
-      $ minio config {{.Name}} add mongo localhost:28710 mydb mylogger
-
-   2. Configure new syslog logger. NOTE: syslog logger is not supported on windows.
-      $ minio config {{.Name}} add syslog localhost:554 udp
-
-   3. Configure new file logger. "/var/log" should be writable by user.
-      $ minio config {{.Name}} add file /var/log/minio.log
-
-   4. List currently configured logger.
-      $ minio config {{.Name}} list
-
-   5. Remove/Reset a configured logger.
-      $ minio config {{.Name}} remove mongo
-`,
-}
-
-// Inherit at one place
-type config struct {
-	*configV2
-}
-
-func mainConfigLogger(ctx *cli.Context) {
-	if !ctx.Args().Present() || ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, "logger", 1) // last argument is exit code
-	}
-	conf, err := loadConfigV2()
-	fatalIf(err.Trace(), "Unable to load config", nil)
-
-	if ctx.Args().Get(0) == "add" {
-		args := ctx.Args().Tail()
-		if args.Get(0) == "mongo" {
-			enableLog2Mongo(&config{conf}, args.Tail())
-		}
-		if args.Get(0) == "syslog" {
-			if runtime.GOOS == "windows" {
-				fatalIf(probe.NewError(errInvalidArgument), "Syslog is not supported on windows.", nil)
-			}
-			enableLog2Syslog(&config{conf}, args.Tail())
-		}
-		if args.Get(0) == "file" {
-			enableLog2File(&config{conf}, args.Tail())
-		}
-	}
-	if ctx.Args().Get(0) == "remove" {
-		args := ctx.Args().Tail()
-		if args.Get(0) == "mongo" {
-			conf.MongoLogger.Addr = ""
-			conf.MongoLogger.DB = ""
-			conf.MongoLogger.Collection = ""
-			err := saveConfig(conf)
-			fatalIf(err.Trace(), "Unable to save config.", nil)
-		}
-		if args.Get(0) == "syslog" {
-			if runtime.GOOS == "windows" {
-				fatalIf(probe.NewError(errInvalidArgument), "Syslog is not supported on windows.", nil)
-			}
-			conf.SyslogLogger.Network = ""
-			conf.SyslogLogger.Addr = ""
-			err := saveConfig(conf)
-			fatalIf(err.Trace(), "Unable to save config.", nil)
-		}
-		if args.Get(0) == "file" {
-			conf.FileLogger.Filename = ""
-			err := saveConfig(conf)
-			fatalIf(err.Trace(), "Unable to save config.", nil)
-		}
-	}
-	if ctx.Args().Get(0) == "list" {
-		console.Println(conf)
-	}
-}
-
-func enableLog2Mongo(conf *config, args cli.Args) {
-	if conf.IsFileLoggingEnabled() {
-		console.Infoln("File logging already enabled. Removing automatically by enabling mongo.")
-		conf.FileLogger.Filename = ""
-	}
-	if conf.IsSysloggingEnabled() {
-		console.Infoln("Syslog logging already enabled. Removing automatically by enabling mongo.")
-		conf.SyslogLogger.Addr = ""
-		conf.SyslogLogger.Network = ""
-	}
-	conf.MongoLogger.Addr = args.Get(0)
-	conf.MongoLogger.DB = args.Get(1)
-	conf.MongoLogger.Collection = args.Get(2)
-
-	err := saveConfig(conf.configV2)
-	fatalIf(err.Trace(), "Unable to save mongo logging config.", nil)
-}
-
-func enableLog2Syslog(conf *config, args cli.Args) {
-	if conf.IsFileLoggingEnabled() {
-		console.Infoln("File logging already enabled. Removing automatically by enabling syslog.")
-		conf.FileLogger.Filename = ""
-	}
-	if conf.IsMongoLoggingEnabled() {
-		console.Infoln("Mongo logging already enabled. Removing automatically by enabling syslog.")
-		conf.MongoLogger.Addr = ""
-		conf.MongoLogger.DB = ""
-		conf.MongoLogger.Collection = ""
-	}
-	conf.SyslogLogger.Addr = args.Get(0)
-	conf.SyslogLogger.Network = args.Get(1)
-	err := saveConfig(conf.configV2)
-	fatalIf(err.Trace(), "Unable to save syslog config.", nil)
-}
-
-func enableLog2File(conf *config, args cli.Args) {
-	if conf.IsSysloggingEnabled() {
-		console.Infoln("Syslog logging already enabled. Removing automatically by enabling file logging.")
-		conf.SyslogLogger.Addr = ""
-		conf.SyslogLogger.Network = ""
-	}
-	if conf.IsMongoLoggingEnabled() {
-		console.Infoln("Mongo logging already enabled. Removing automatically by enabling file logging.")
-		conf.MongoLogger.Addr = ""
-		conf.MongoLogger.DB = ""
-		conf.MongoLogger.Collection = ""
-	}
-	conf.FileLogger.Filename = args.Get(0)
-	err := saveConfig(conf.configV2)
-	fatalIf(err.Trace(), "Unable to save file logging config.", nil)
-}

config-main.go

@@ -1,53 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "github.com/minio/cli"
-
-//   Configure minio server
-//
-//   ----
-//   NOTE: that the configure command only writes values to the config file.
-//   It does not use any configuration values from the environment variables.
-//   ----
-//
-var configCmd = cli.Command{
-	Name:   "config",
-	Usage:  "Collection of config management commands.",
-	Action: mainConfig,
-	Subcommands: []cli.Command{
-		configLoggerCmd,
-		configVersionCmd,
-	},
-	CustomHelpTemplate: `NAME:
-  {{.Name}} - {{.Usage}}
-
-USAGE:
-  {{.Name}} {{if .Flags}}[global flags] {{end}}command{{if .Flags}} [command flags]{{end}} [arguments...]
-
-COMMANDS:
-  {{range .Commands}}{{ .Name }}{{ "\t" }}{{.Usage}}
-  {{end}}
-`,
-}
-
-// mainConfig is the handle for "minio config" command. provides sub-commands which write configuration data in json format to config file.
-func mainConfig(ctx *cli.Context) {
-	if !ctx.Args().Present() || ctx.Args().First() == "help" {
-		cli.ShowAppHelp(ctx)
-	}
-}

config-migrate.go

@@ -0,0 +1,147 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/quick"
+)
+
+func migrateConfig() {
+	// Purge all configs with version '1'.
+	purgeV1()
+	// Migrate version '2' to '3'.
+	migrateV2ToV3()
+	// Migrate version '3' to '4'.
+	migrateV3ToV4()
+}
+
+// Version '1' is not supported anymore and deprecated, safe to delete.
+func purgeV1() {
+	cv1, err := loadConfigV1()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return
+		}
+	}
+	fatalIf(err.Trace(), "Unable to load config version ‘1’.", nil)
+
+	if cv1.Version == "1" {
+		console.Println("Unsupported config version ‘1’ found, removed successfully.")
+		/// Purge old fsUsers.json file
+		configPath, err := getConfigPath()
+		fatalIf(err.Trace(), "Unable to retrieve config path.", nil)
+
+		configFile := filepath.Join(configPath, "fsUsers.json")
+		os.RemoveAll(configFile)
+	}
+	fatalIf(probe.NewError(errors.New("")), "Unexpected version found ‘"+cv1.Version+"’, cannot migrate.", nil)
+}
+
+// Version '2' to '3' config migration adds new fields and re-orders
+// previous fields. Simplifies config for future additions.
+func migrateV2ToV3() {
+	cv2, err := loadConfigV2()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return
+		}
+	}
+	fatalIf(err.Trace(), "Unable to load config version ‘2’.", nil)
+	if cv2.Version != "2" {
+		return
+	}
+	srvConfig := &configV3{}
+	srvConfig.Version = "3"
+	srvConfig.Addr = ":9000"
+	srvConfig.Credential = credential{
+		AccessKeyID:     cv2.Credentials.AccessKeyID,
+		SecretAccessKey: cv2.Credentials.SecretAccessKey,
+	}
+	srvConfig.Region = cv2.Credentials.Region
+	srvConfig.Logger.Console = consoleLogger{
+		Enable: true,
+		Level:  "fatal",
+	}
+	flogger := fileLogger{}
+	flogger.Level = "error"
+	if cv2.FileLogger.Filename != "" {
+		flogger.Enable = true
+		flogger.Filename = cv2.FileLogger.Filename
+	}
+	srvConfig.Logger.File = flogger
+
+	slogger := syslogLogger{}
+	slogger.Level = "debug"
+	if cv2.SyslogLogger.Addr != "" {
+		slogger.Enable = true
+		slogger.Addr = cv2.SyslogLogger.Addr
+	}
+	srvConfig.Logger.Syslog = slogger
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err.Trace(), "Unable to initialize config.", nil)
+
+	configFile, err := getConfigFile()
+	fatalIf(err.Trace(), "Unable to get config file.", nil)
+
+	// Migrate the config.
+	err = qc.Save(configFile)
+	fatalIf(err.Trace(), "Migrating from version ‘"+cv2.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil)
+
+	console.Println("Migration from version ‘" + cv2.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}
+
+// Version '3' to '4' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV3ToV4() {
+	cv3, err := loadConfigV3()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return
+		}
+	}
+	fatalIf(err.Trace(), "Unable to load config version ‘3’.", nil)
+	if cv3.Version != "3" {
+		return
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &serverConfigV4{}
+	srvConfig.Version = globalMinioConfigVersion
+	srvConfig.Credential = cv3.Credential
+	srvConfig.Region = cv3.Region
+	srvConfig.Logger.Console = cv3.Logger.Console
+	srvConfig.Logger.File = cv3.Logger.File
+	srvConfig.Logger.Syslog = cv3.Logger.Syslog
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err.Trace(), "Unable to initialize the quick config.", nil)
+	configFile, err := getConfigFile()
+	fatalIf(err.Trace(), "Unable to get config file.", nil)
+
+	err = qc.Save(configFile)
+	fatalIf(err.Trace(), "Migrating from version ‘"+cv3.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil)
+
+	console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}

config-old.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/quick"
+)
+
+/////////////////// Config V1 ///////////////////
+type configV1 struct {
+	Version         string `json:"version"`
+	AccessKeyID     string `json:"accessKeyId"`
+	SecretAccessKey string `json:"secretAccessKey"`
+}
+
+// loadConfigV1 load config
+func loadConfigV1() (*configV1, *probe.Error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	configFile := filepath.Join(configPath, "fsUsers.json")
+	if _, err := os.Stat(configFile); err != nil {
+		return nil, probe.NewError(err)
+	}
+	a := &configV1{}
+	a.Version = "1"
+	qc, err := quick.New(a)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*configV1), nil
+}
+
+/////////////////// Config V2 ///////////////////
+type configV2 struct {
+	Version     string `json:"version"`
+	Credentials struct {
+		AccessKeyID     string `json:"accessKeyId"`
+		SecretAccessKey string `json:"secretAccessKey"`
+		Region          string `json:"region"`
+	} `json:"credentials"`
+	MongoLogger struct {
+		Addr       string `json:"addr"`
+		DB         string `json:"db"`
+		Collection string `json:"collection"`
+	} `json:"mongoLogger"`
+	SyslogLogger struct {
+		Network string `json:"network"`
+		Addr    string `json:"addr"`
+	} `json:"syslogLogger"`
+	FileLogger struct {
+		Filename string `json:"filename"`
+	} `json:"fileLogger"`
+}
+
+// loadConfigV2 load config version '2'.
+func loadConfigV2() (*configV2, *probe.Error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if _, err := os.Stat(configFile); err != nil {
+		return nil, probe.NewError(err)
+	}
+	a := &configV2{}
+	a.Version = "2"
+	qc, err := quick.New(a)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*configV2), nil
+}
+
+/////////////////// Config V3 ///////////////////
+
+// backendV3 type.
+type backendV3 struct {
+	Type  string   `json:"type"`
+	Disk  string   `json:"disk,omitempty"`
+	Disks []string `json:"disks,omitempty"`
+}
+
+// loggerV3 type.
+type loggerV3 struct {
+	Console struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+	}
+	File struct {
+		Enable   bool   `json:"enable"`
+		Filename string `json:"fileName"`
+		Level    string `json:"level"`
+	}
+	Syslog struct {
+		Enable bool   `json:"enable"`
+		Addr   string `json:"address"`
+		Level  string `json:"level"`
+	} `json:"syslog"`
+	// Add new loggers here.
+}
+
+// configV3 server configuration version '3'.
+type configV3 struct {
+	Version string `json:"version"`
+
+	// Backend configuration.
+	Backend backendV3 `json:"backend"`
+
+	// http Server configuration.
+	Addr string `json:"address"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV3 `json:"logger"`
+}
+
+// loadConfigV3 load config version '3'.
+func loadConfigV3() (*configV3, *probe.Error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if _, err := os.Stat(configFile); err != nil {
+		return nil, probe.NewError(err)
+	}
+	a := &configV3{}
+	a.Version = "3"
+	qc, err := quick.New(a)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*configV3), nil
+}

config-v4.go

@@ -0,0 +1,207 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"sync"
+
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/quick"
+)
+
+// serverConfigV4 server configuration version '4'.
+type serverConfigV4 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger logger `json:"logger"`
+
+	// Read Write mutex.
+	rwMutex *sync.RWMutex
+}
+
+// initConfig - initialize server config. config version (called only once).
+func initConfig() *probe.Error {
+	if !isConfigFileExists() {
+		srvCfg := &serverConfigV4{}
+		srvCfg.Version = globalMinioConfigVersion
+		srvCfg.Region = "us-east-1"
+		srvCfg.Credential = mustGenAccessKeys()
+		// Enable console logger by default on a fresh run.
+		srvCfg.Logger.Console = consoleLogger{
+			Enable: true,
+			Level:  "fatal",
+		}
+		srvCfg.rwMutex = &sync.RWMutex{}
+		// Create config path.
+		err := createConfigPath()
+		if err != nil {
+			return err.Trace()
+		}
+
+		// Create certs path.
+		err = createCertsPath()
+		if err != nil {
+			return err.Trace()
+		}
+
+		// Save the new config globally.
+		serverConfig = srvCfg
+
+		// Save config into file.
+		err = serverConfig.Save()
+		if err != nil {
+			return err.Trace()
+		}
+		return nil
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err.Trace()
+	}
+	if _, e := os.Stat(configFile); err != nil {
+		return probe.NewError(e)
+	}
+	srvCfg := &serverConfigV4{}
+	srvCfg.Version = globalMinioConfigVersion
+	srvCfg.rwMutex = &sync.RWMutex{}
+	qc, err := quick.New(srvCfg)
+	if err != nil {
+		return err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return err.Trace()
+	}
+	// Save the loaded config globally.
+	serverConfig = qc.Data().(*serverConfigV4)
+	// Set the version properly after the unmarshalled json is loaded.
+	serverConfig.Version = globalMinioConfigVersion
+	return nil
+}
+
+// serverConfig server config.
+var serverConfig *serverConfigV4
+
+// GetVersion get current config version.
+func (s serverConfigV4) GetVersion() string {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Version
+}
+
+/// Logger related.
+
+// SetFileLogger set new file logger.
+func (s *serverConfigV4) SetFileLogger(flogger fileLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.File = flogger
+}
+
+// GetFileLogger get current file logger.
+func (s serverConfigV4) GetFileLogger() fileLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.File
+}
+
+// SetConsoleLogger set new console logger.
+func (s *serverConfigV4) SetConsoleLogger(clogger consoleLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.Console = clogger
+}
+
+// GetConsoleLogger get current console logger.
+func (s serverConfigV4) GetConsoleLogger() consoleLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.Console
+}
+
+// SetSyslogLogger set new syslog logger.
+func (s *serverConfigV4) SetSyslogLogger(slogger syslogLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.Syslog = slogger
+}
+
+// GetSyslogLogger get current syslog logger.
+func (s *serverConfigV4) GetSyslogLogger() syslogLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.Syslog
+}
+
+// SetRegion set new region.
+func (s *serverConfigV4) SetRegion(region string) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Region = region
+}
+
+// GetRegion get current region.
+func (s serverConfigV4) GetRegion() string {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Region
+}
+
+// SetCredentials set new credentials.
+func (s *serverConfigV4) SetCredential(creds credential) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Credential = creds
+}
+
+// GetCredentials get current credentials.
+func (s serverConfigV4) GetCredential() credential {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Credential
+}
+
+// Save config.
+func (s serverConfigV4) Save() *probe.Error {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+
+	// get config file.
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err.Trace()
+	}
+
+	// initialize quick.
+	qc, err := quick.New(&s)
+	if err != nil {
+		return err.Trace()
+	}
+
+	// Save config file.
+	if err := qc.Save(configFile); err != nil {
+		return err.Trace()
+	}
+
+	// Return success.
+	return nil
+}

config-version-main.go

@@ -1,50 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"github.com/minio/cli"
-	"github.com/minio/mc/pkg/console"
-)
-
-// Print config version.
-var configVersionCmd = cli.Command{
-	Name:   "version",
-	Usage:  "Print config version.",
-	Action: mainConfigVersion,
-	CustomHelpTemplate: `NAME:
-   minio config {{.Name}} - {{.Usage}}
-
-USAGE:
-   minio config {{.Name}}
-
-`,
-}
-
-func mainConfigVersion(ctx *cli.Context) {
-	if ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, "version", 1) // last argument is exit code
-	}
-
-	config, err := loadConfigV2()
-	fatalIf(err.Trace(), "Unable to load config", nil)
-
-	// convert interface{} back to its original struct
-	newConf := config
-	type Version string
-	console.Println(newConf.Version)
-}

config.go

@@ -0,0 +1,92 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/go-homedir"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// configPath for custom config path only for testing purposes
+var customConfigPath string
+
+// Sets a new config path.
+func setGlobalConfigPath(configPath string) {
+	customConfigPath = configPath
+}
+
+// getConfigPath get server config path
+func getConfigPath() (string, *probe.Error) {
+	if customConfigPath != "" {
+		return customConfigPath, nil
+	}
+	homeDir, e := homedir.Dir()
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+	configPath := filepath.Join(homeDir, globalMinioConfigDir)
+	return configPath, nil
+}
+
+// mustGetConfigPath must get server config path.
+func mustGetConfigPath() string {
+	configPath, err := getConfigPath()
+	fatalIf(err.Trace(), "Unable to get config path.", nil)
+	return configPath
+}
+
+// createConfigPath create server config path.
+func createConfigPath() *probe.Error {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return err.Trace()
+	}
+	if err := os.MkdirAll(configPath, 0700); err != nil {
+		return probe.NewError(err)
+	}
+	return nil
+}
+
+// isConfigFileExists - returns true if config file exists.
+func isConfigFileExists() bool {
+	st, e := os.Stat(mustGetConfigFile())
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// mustGetConfigFile must get server config file.
+func mustGetConfigFile() string {
+	configFile, err := getConfigFile()
+	fatalIf(err.Trace(), "Unable to get config file.", nil)
+
+	return configFile
+}
+
+// getConfigFile get server config file.
+func getConfigFile() (string, *probe.Error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err.Trace()
+	}
+	return filepath.Join(configPath, globalMinioConfigFile), nil
+}

docker/Dockerfile

@@ -1,8 +1,8 @@
 # use "make dockerimage" to build
 FROM alpine:3.3
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates bash
 ADD minio.dockerimage /minio
 ADD export /export
 EXPOSE 9000
-ENTRYPOINT ["/minio"]
-CMD ["server", "/export"]
+COPY start.sh /start.sh
+ENTRYPOINT ["/start.sh"]

docker/start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+## Fix this check when we arrive at XL.
+if [ -z "$1" ]; then
+    echo "Invalid arguments"
+    echo "Usage: <export_dir>"
+    exit 1
+else
+    /bin/mkdir -p "$1" && /minio server "$1"
+fi

flags.go

@@ -1,57 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "github.com/minio/cli"
-
-// Collection of minio flags currently supported
-var flags = []cli.Flag{}
-
-var (
-	configFolderFlag = cli.StringFlag{
-		Name:  "config-folder, C",
-		Value: mustGetConfigPath(),
-		Usage: "Path to configuration folder.",
-	}
-
-	addressFlag = cli.StringFlag{
-		Name:  "address",
-		Value: ":9000",
-		Usage: "ADDRESS:PORT for cloud storage access.",
-	}
-
-	accessLogFlag = cli.BoolFlag{
-		Name:  "enable-accesslog",
-		Hide:  true,
-		Usage: "Enable access logs for all incoming HTTP request.",
-	}
-
-	certFlag = cli.StringFlag{
-		Name:  "cert",
-		Usage: "Provide your domain certificate.",
-	}
-
-	keyFlag = cli.StringFlag{
-		Name:  "key",
-		Usage: "Provide your domain private key.",
-	}
-)
-
-// registerFlag registers a cli flag
-func registerFlag(flag cli.Flag) {
-	flags = append(flags, flag)
-}

fs-bucket-listobjects.go

@@ -0,0 +1,182 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+const (
+	// listObjectsLimit - maximum list objects limit.
+	listObjectsLimit = 1000
+)
+
+// isDirExist - returns whether given directory is exist or not.
+func isDirExist(dirname string) (bool, error) {
+	fi, e := os.Lstat(dirname)
+	if e != nil {
+		if os.IsNotExist(e) {
+			return false, nil
+		}
+		return false, e
+	}
+	return fi.IsDir(), nil
+}
+
+func (fs *Filesystem) saveTreeWalk(params listObjectParams, walker *treeWalker) {
+	fs.listObjectMapMutex.Lock()
+	defer fs.listObjectMapMutex.Unlock()
+
+	walkers, _ := fs.listObjectMap[params]
+	walkers = append(walkers, walker)
+
+	fs.listObjectMap[params] = walkers
+}
+
+func (fs *Filesystem) lookupTreeWalk(params listObjectParams) *treeWalker {
+	fs.listObjectMapMutex.Lock()
+	defer fs.listObjectMapMutex.Unlock()
+
+	if walkChs, ok := fs.listObjectMap[params]; ok {
+		for i, walkCh := range walkChs {
+			if !walkCh.timedOut {
+				newWalkChs := walkChs[i+1:]
+				if len(newWalkChs) > 0 {
+					fs.listObjectMap[params] = newWalkChs
+				} else {
+					delete(fs.listObjectMap, params)
+				}
+				return walkCh
+			}
+		}
+		// As all channels are timed out, delete the map entry
+		delete(fs.listObjectMap, params)
+	}
+	return nil
+}
+
+// ListObjects - lists all objects for a given prefix, returns up to
+// maxKeys number of objects per call.
+func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error) {
+	result := ListObjectsInfo{}
+
+	// Input validation.
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return result, probe.NewError(e)
+	}
+	bucketDir := filepath.Join(fs.diskPath, bucket)
+
+	if !IsValidObjectPrefix(prefix) {
+		return result, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix})
+	}
+
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported. Only '/' is supported", delimiter))
+	}
+
+	// Verify if marker has prefix.
+	if marker != "" {
+		if !strings.HasPrefix(marker, prefix) {
+			return result, probe.NewError(fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", marker, prefix))
+		}
+	}
+
+	// Return empty response for a valid request when maxKeys is 0.
+	if maxKeys == 0 {
+		return result, nil
+	}
+
+	// Over flowing maxkeys - reset to listObjectsLimit.
+	if maxKeys < 0 || maxKeys > listObjectsLimit {
+		maxKeys = listObjectsLimit
+	}
+
+	// Verify if prefix exists.
+	prefixDir := filepath.Dir(filepath.FromSlash(prefix))
+	rootDir := filepath.Join(bucketDir, prefixDir)
+	if status, e := isDirExist(rootDir); !status {
+		if e == nil {
+			// Prefix does not exist, not an error just respond empty
+			// list response.
+			return result, nil
+		}
+		// Rest errors should be treated as failure.
+		return result, probe.NewError(e)
+	}
+
+	recursive := true
+	if delimiter == "/" {
+		recursive = false
+	}
+
+	// Maximum 1000 objects returned in a single to listObjects.
+	// Further calls will set right marker value to continue reading the rest of the objectList.
+	// popTreeWalker returns nil if the call to ListObject is done for the first time.
+	// On further calls to ListObjects to retrive more objects within the timeout period,
+	// popTreeWalker returns the channel from which rest of the objects can be retrieved.
+	walker := fs.lookupTreeWalk(listObjectParams{bucket, delimiter, marker, prefix})
+	if walker == nil {
+		walker = startTreeWalk(fs.diskPath, bucket, filepath.FromSlash(prefix), filepath.FromSlash(marker), recursive)
+	}
+
+	nextMarker := ""
+	for i := 0; i < maxKeys; {
+		walkResult, ok := <-walker.ch
+		if !ok {
+			// Closed channel.
+			return result, nil
+		}
+		// For any walk error return right away.
+		if walkResult.err != nil {
+			return ListObjectsInfo{}, probe.NewError(walkResult.err)
+		}
+		objInfo := walkResult.objectInfo
+		objInfo.Name = filepath.ToSlash(objInfo.Name)
+
+		// Skip temporary files.
+		if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") {
+			continue
+		}
+
+		// For objects being directory and delimited we set Prefixes.
+		if objInfo.IsDir {
+			result.Prefixes = append(result.Prefixes, objInfo.Name)
+		} else {
+			result.Objects = append(result.Objects, objInfo)
+		}
+
+		// We have listed everything return.
+		if walkResult.end {
+			return result, nil
+		}
+		nextMarker = objInfo.Name
+		i++
+	}
+	// We haven't exhaused the list yet, set IsTruncated to 'true' so
+	// that the client can send another request.
+	result.IsTruncated = true
+	result.NextMarker = nextMarker
+	fs.saveTreeWalk(listObjectParams{bucket, delimiter, nextMarker, prefix}, walker)
+	return result, nil
+}

fs-bucket-listobjects_test.go

@@ -0,0 +1,604 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestListObjects(t *testing.T) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-list-object-test")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// This bucket is used for testing ListObject operations.
+	err = fs.MakeBucket("test-bucket-list-object")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Will not store any objects in this bucket,
+	// Its to test ListObjects on an empty bucket.
+	err = fs.MakeBucket("empty-bucket")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tmpfile, e := ioutil.TempFile("", "simple-file.txt")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	_, err = fs.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil)
+	if err != nil {
+		t.Fatal(e)
+	}
+
+	_, err = fs.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
+	if err != nil {
+		t.Fatal(e)
+	}
+
+	_, err = fs.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
+	if err != nil {
+		t.Fatal(e)
+	}
+
+	for i := 0; i < 2; i++ {
+		key := "newPrefix" + strconv.Itoa(i)
+		_, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	_, err = fs.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil)
+	if err != nil {
+		t.Fatal(e)
+	}
+
+	for i := 0; i < 3; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Formualting the result data set to be expected from ListObjects call inside the tests,
+	// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
+
+	resultCases := []ListObjectsInfo{
+		// ListObjectsResult-0.
+		// Testing for listing all objects in the bucket, (testCase 20,21,22).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-1.
+		// Used for asserting the truncated case, (testCase 23).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+			},
+		},
+		// ListObjectsResult-2.
+		// (TestCase 24).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+			},
+		},
+		// ListObjectsResult-3.
+		// (TestCase 25).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+		// ListObjectsResult-4.
+		// Again used for truncated case.
+		// (TestCase 26).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+			},
+		},
+		// ListObjectsResult-5.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix "new", (testCase 27-29).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-6.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix = "obj", (testCase 30).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-7.
+		// Used for Asserting prefixes and truncation.
+		// Used for test case with prefix = "new" and maxKeys = 1, (testCase 31).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+			},
+		},
+		// ListObjectsResult-8.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "obj0"},
+				{Name: "obj1"},
+			},
+		},
+		// ListObjectsResult-9.
+		// Used for asserting the case with marker, but without prefix.
+		//marker is set to "newPrefix0" in the testCase, (testCase 33).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-10.
+		//marker is set to "newPrefix1" in the testCase, (testCase 34).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-11.
+		//marker is set to "obj0" in the testCase, (testCase 35).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-12.
+		// Marker is set to "obj1" in the testCase, (testCase 36).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-13.
+		// Marker is set to "man" in the testCase, (testCase37).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-14.
+		// Marker is set to "Abc" in the testCase, (testCase 39).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-15.
+		// Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-16.
+		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-17.
+		// Used for asserting the case with marker, without prefix but with truncation.
+		// Marker =  "newPrefix0" & maxKeys = 3 in the testCase, (testCase42).
+		// Output truncated to 3 values.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+			},
+		},
+		// ListObjectsResult-18.
+		// Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43).
+		// Output truncated to 1 value.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-19.
+		// Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44).
+		// Output truncated to 1 value.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+			},
+		},
+		// ListObjectsResult-20.
+		// Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-21.
+		// Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-22.
+		// Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-23.
+		// Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+
+		// ListObjectsResult-24.
+		// Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+
+		// ListObjectsResult-25.
+		// Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+			},
+		},
+		// ListObjectsResult-26.
+		// prefix = "new" and delimiter is set in the testCase.(testCase 58).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+			},
+		},
+		// ListObjectsResult-27.
+		// Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/India-summer-photos-1"},
+			},
+		},
+		// ListObjectsResult-28.
+		// Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-29.
+		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimeter set, (testCase 61).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+	}
+
+	testCases := []struct {
+		// Inputs to ListObjects.
+		bucketName string
+		prefix     string
+		marker     string
+		delimeter  string
+		maxKeys    int
+		// Expected output of ListObjects.
+		result ListObjectsInfo
+		err    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names ( Test number 1-4 ).
+		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Using an existing file for bucket name, but its not a directory (5).
+		{"simple-file.txt", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "simple-file.txt"}, false},
+		// Valid bucket names, but they donot exist (6-8).
+		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		// Valid, existing bucket, but sending invalid delimeter values (9-10).
+		// Empty string < "" > and forward slash < / > are the ony two valid arguments for delimeter.
+		{"test-bucket-list-object", "", "", "*", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "*"), false},
+		{"test-bucket-list-object", "", "", "-", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "-"), false},
+		// Testing for failure cases with both perfix and marker (13).
+		// The prefix and marker combination to be valid it should satisy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
+		// Setting a non-existing directory to be prefix (14-15).
+		{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
+		{"empty-bucket", "europe/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
+		// Testing on empty bucket, that is, bucket without any objects in it (16).
+		{"empty-bucket", "", "", "", 0, ListObjectsInfo{}, nil, true},
+		// Setting maxKeys to negative value (17-18).
+		{"empty-bucket", "", "", "", -1, ListObjectsInfo{}, nil, true},
+		{"empty-bucket", "", "", "", 1, ListObjectsInfo{}, nil, true},
+		// Setting maxKeys to a very large value (19).
+		{"empty-bucket", "", "", "", 1111000000000000, ListObjectsInfo{}, nil, true},
+		// Testing for all 7 objects in the bucket (20).
+		{"test-bucket-list-object", "", "", "", 9, resultCases[0], nil, true},
+		//Testing for negative value of maxKey, this should set maxKeys to listObjectsLimit (21).
+		{"test-bucket-list-object", "", "", "", -1, resultCases[0], nil, true},
+		// Testing for very large value of maxKey, this should set maxKeys to listObjectsLimit (22).
+		{"test-bucket-list-object", "", "", "", 1234567891011, resultCases[0], nil, true},
+		// Testing for trancated value (23-26).
+		{"test-bucket-list-object", "", "", "", 5, resultCases[1], nil, true},
+		{"test-bucket-list-object", "", "", "", 4, resultCases[2], nil, true},
+		{"test-bucket-list-object", "", "", "", 3, resultCases[3], nil, true},
+		{"test-bucket-list-object", "", "", "", 1, resultCases[4], nil, true},
+		// Testing with prefix (27-30).
+		{"test-bucket-list-object", "new", "", "", 3, resultCases[5], nil, true},
+		{"test-bucket-list-object", "new", "", "", 4, resultCases[5], nil, true},
+		{"test-bucket-list-object", "new", "", "", 5, resultCases[5], nil, true},
+		{"test-bucket-list-object", "obj", "", "", 3, resultCases[6], nil, true},
+		// Testing with prefix and truncation (31-32).
+		{"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true},
+		{"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true},
+		// Testing with marker, but without prefix and truncation (33-37).
+		{"test-bucket-list-object", "", "newPrefix0", "", 5, resultCases[9], nil, true},
+		{"test-bucket-list-object", "", "newPrefix1", "", 4, resultCases[10], nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 2, resultCases[11], nil, true},
+		{"test-bucket-list-object", "", "obj1", "", 1, resultCases[12], nil, true},
+		{"test-bucket-list-object", "", "man", "", 10, resultCases[13], nil, true},
+		// Marker being set to a value which is greater than and all object names when sorted (38).
+		// Expected to send an empty response in this case.
+		{"test-bucket-list-object", "", "zen", "", 10, ListObjectsInfo{}, nil, true},
+		// Marker being set to a value which is lesser than and all object names when sorted (39).
+		// Expected to send all the objects in the bucket in this case.
+		{"test-bucket-list-object", "", "Abc", "", 10, resultCases[14], nil, true},
+		// Marker is to a hierarhical value (40-41).
+		{"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "", 10, resultCases[15], nil, true},
+		{"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", 10, resultCases[16], nil, true},
+		// Testing with marker and truncation, but no prefix (42-44).
+		{"test-bucket-list-object", "", "newPrefix0", "", 3, resultCases[17], nil, true},
+		{"test-bucket-list-object", "", "newPrefix1", "", 1, resultCases[18], nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 1, resultCases[19], nil, true},
+		// Testing with both marker and prefix, but without truncation (45-47).
+		// The valid combination of marker and prefix should satisfy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "obj", "obj0", "", 2, resultCases[20], nil, true},
+		{"test-bucket-list-object", "obj", "obj1", "", 1, resultCases[21], nil, true},
+		{"test-bucket-list-object", "new", "newPrefix0", "", 2, resultCases[22], nil, true},
+		// Testing with maxKeys set to 0 (48-54).
+		// The parameters have to valid.
+		{"test-bucket-list-object", "", "obj1", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "new", "", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "obj0", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "obj1", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "new", "newPrefix0", "", 0, ListObjectsInfo{}, nil, true},
+		// Tests on hierarchical key names as prefix.
+		// Without delimteter the code should recurse into the prefix Dir.
+		// Tests with prefix, but without delimiter (55-56).
+		{"test-bucket-list-object", "Asia/India/", "", "", 10, resultCases[23], nil, true},
+		{"test-bucket-list-object", "Asia", "", "", 10, resultCases[24], nil, true},
+		// Tests with prefix and delimiter (57-59).
+		// With delimeter the code shouldnot recurse into the sub-directories of prefix Dir.
+		{"test-bucket-list-object", "Asia", "", "/", 10, resultCases[25], nil, true},
+		{"test-bucket-list-object", "new", "", "/", 10, resultCases[26], nil, true},
+		{"test-bucket-list-object", "Asia/India/", "", "/", 10, resultCases[27], nil, true},
+		// Test with marker set as hierarhical value and with delimiter. (60-61)
+		{"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "/", 10, resultCases[28], nil, true},
+		{"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "/", 10, resultCases[29], nil, true},
+	}
+
+	for i, testCase := range testCases {
+		result, err := fs.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if !strings.Contains(err.Cause.Error(), testCase.err.Error()) {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error())
+			}
+		}
+		// Since there are cases for which ListObjects fails, this is necessary.
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			// The length of the expected ListObjectsResult.Objects should match in both expected result from test cases and in the output.
+			// On failure calling t.Fatalf, otherwise it may lead to index out of range error in assertion following this.
+			if len(testCase.result.Objects) != len(result.Objects) {
+				t.Fatalf("Test %d: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, len(testCase.result.Objects), len(result.Objects))
+			}
+			for j := 0; j < len(testCase.result.Objects); j++ {
+				if testCase.result.Objects[j].Name != result.Objects[j].Name {
+					t.Errorf("Test %d: Expected object name to be \"%s\", but found \"%s\" instead", i+1, testCase.result.Objects[j].Name, result.Objects[j].Name)
+				}
+			}
+			if testCase.result.IsTruncated != result.IsTruncated {
+				t.Errorf("Test %d: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, testCase.result.IsTruncated, result.IsTruncated)
+			}
+
+		}
+	}
+
+}
+
+func BenchmarkListObjects(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-list-benchmark")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Create a bucket.
+	err = fs.MakeBucket("ls-benchmark-bucket")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	for i := 0; i < 20000; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+
+	// List the buckets over and over and over.
+	for i := 0; i < b.N; i++ {
+		_, err = fs.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

fs-bucket.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package fs
+package main
 
 import (
 	"io/ioutil"
@@ -22,7 +22,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/probe"
 )
 
@@ -34,8 +33,8 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
 	if !IsValidBucketName(bucket) {
 		return probe.NewError(BucketNameInvalid{Bucket: bucket})
 	}
-	bucket = fs.denormalizeBucket(bucket)
-	bucketDir := filepath.Join(fs.path, bucket)
+	bucket = getActualBucketname(fs.diskPath, bucket)
+	bucketDir := filepath.Join(fs.diskPath, bucket)
 	if e := os.Remove(bucketDir); e != nil {
 		// Error if there was no bucket in the first place.
 		if os.IsNotExist(e) {
@@ -56,12 +55,12 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
 }
 
 // ListBuckets - Get service.
-func (fs Filesystem) ListBuckets() ([]BucketMetadata, *probe.Error) {
-	files, e := ioutil.ReadDir(fs.path)
+func (fs Filesystem) ListBuckets() ([]BucketInfo, *probe.Error) {
+	files, e := ioutil.ReadDir(fs.diskPath)
 	if e != nil {
-		return []BucketMetadata{}, probe.NewError(e)
+		return []BucketInfo{}, probe.NewError(e)
 	}
-	var metadataList []BucketMetadata
+	var buckets []BucketInfo
 	for _, file := range files {
 		if !file.IsDir() {
 			// If not directory, ignore all file types.
@@ -72,19 +71,19 @@ func (fs Filesystem) ListBuckets() ([]BucketMetadata, *probe.Error) {
 		if !IsValidBucketName(dirName) {
 			continue
 		}
-		metadata := BucketMetadata{
+		bucket := BucketInfo{
 			Name:    dirName,
 			Created: file.ModTime(),
 		}
-		metadataList = append(metadataList, metadata)
+		buckets = append(buckets, bucket)
 	}
 	// Remove duplicated entries.
-	metadataList = removeDuplicateBuckets(metadataList)
-	return metadataList, nil
+	buckets = removeDuplicateBuckets(buckets)
+	return buckets, nil
 }
 
 // removeDuplicateBuckets - remove duplicate buckets.
-func removeDuplicateBuckets(buckets []BucketMetadata) []BucketMetadata {
+func removeDuplicateBuckets(buckets []BucketInfo) []BucketInfo {
 	length := len(buckets) - 1
 	for i := 0; i < length; i++ {
 		for j := i + 1; j <= length; j++ {
@@ -105,71 +104,59 @@ func removeDuplicateBuckets(buckets []BucketMetadata) []BucketMetadata {
 
 // MakeBucket - PUT Bucket
 func (fs Filesystem) MakeBucket(bucket string) *probe.Error {
-	di, err := disk.GetInfo(fs.path)
-	if err != nil {
-		return probe.NewError(err)
-	}
-
-	// Remove 5% from total space for cumulative disk space used for
-	// journalling, inodes etc.
-	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
-	if int64(availableDiskSpace) <= fs.minFreeDisk {
-		return probe.NewError(RootPathFull{Path: fs.path})
-	}
-
-	// Verify if bucket is valid.
-	if !IsValidBucketName(bucket) {
+	if _, e := fs.checkBucketArg(bucket); e == nil {
+		return probe.NewError(BucketExists{Bucket: bucket})
+	} else if _, ok := e.(BucketNameInvalid); ok {
 		return probe.NewError(BucketNameInvalid{Bucket: bucket})
 	}
-
-	bucket = fs.denormalizeBucket(bucket)
-	bucketDir := filepath.Join(fs.path, bucket)
-	if _, e := os.Stat(bucketDir); e == nil {
-		return probe.NewError(BucketExists{Bucket: bucket})
-	}
+	bucketDir := filepath.Join(fs.diskPath, bucket)
 
 	// Make bucket.
 	if e := os.Mkdir(bucketDir, 0700); e != nil {
-		return probe.NewError(err)
+		return probe.NewError(e)
 	}
 	return nil
 }
 
-// denormalizeBucket - will convert incoming bucket names to
-// corresponding valid bucketnames on the backend in a platform
+// getActualBucketname - will convert incoming bucket names to
+// corresponding actual bucketnames on the backend in a platform
 // compatible way for all operating systems.
-func (fs Filesystem) denormalizeBucket(bucket string) string {
-	buckets, e := ioutil.ReadDir(fs.path)
+func getActualBucketname(fsPath, bucket string) string {
+	fd, e := os.Open(fsPath)
+	if e != nil {
+		return bucket
+	}
+	buckets, e := fd.Readdirnames(-1)
 	if e != nil {
 		return bucket
 	}
 	for _, b := range buckets {
-		// Verify if lowercase version of the bucket is equal to the
-		// incoming bucket, then use the proper name.
-		if strings.ToLower(b.Name()) == bucket {
-			return b.Name()
+		// Verify if lowercase version of the bucket is equal
+		// to the incoming bucket, then use the proper name.
+		if strings.ToLower(b) == bucket {
+			return b
 		}
 	}
 	return bucket
 }
 
-// GetBucketMetadata - get bucket metadata.
-func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error) {
+// GetBucketInfo - get bucket metadata.
+func (fs Filesystem) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) {
 	if !IsValidBucketName(bucket) {
-		return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+		return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
 	}
-	bucket = fs.denormalizeBucket(bucket)
+	bucket = getActualBucketname(fs.diskPath, bucket)
 	// Get bucket path.
-	bucketDir := filepath.Join(fs.path, bucket)
+	bucketDir := filepath.Join(fs.diskPath, bucket)
 	fi, e := os.Stat(bucketDir)
 	if e != nil {
 		// Check if bucket exists.
 		if os.IsNotExist(e) {
-			return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+			return BucketInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
 		}
-		return BucketMetadata{}, probe.NewError(e)
+		return BucketInfo{}, probe.NewError(e)
 	}
-	bucketMetadata := BucketMetadata{}
+	bucketMetadata := BucketInfo{}
 	bucketMetadata.Name = fi.Name()
 	bucketMetadata.Created = fi.ModTime()
 	return bucketMetadata, nil

fs-bucket_test.go

@@ -0,0 +1,256 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// The test not just includes asserting the correctness of the output,
+// But also includes test cases for which the function should fail.
+// For those cases for which it fails, its also asserted whether the function fails as expected.
+func TestGetBucketInfo(t *testing.T) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-metadata-test")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Creating few buckets.
+	for i := 0; i < 4; i++ {
+		err = fs.MakeBucket("meta-test-bucket." + strconv.Itoa(i))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	testCases := []struct {
+		bucketName string
+		metaData   BucketInfo
+		e          error
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names.
+		{".test", BucketInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", BucketInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", BucketInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", BucketInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Test cases with non-existent buckets.
+		{"volatile-bucket-1", BucketInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", BucketInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		// Test cases with existing buckets.
+		{"meta-test-bucket.0", BucketInfo{Name: "meta-test-bucket.0"}, nil, true},
+		{"meta-test-bucket.1", BucketInfo{Name: "meta-test-bucket.1"}, nil, true},
+		{"meta-test-bucket.2", BucketInfo{Name: "meta-test-bucket.2"}, nil, true},
+		{"meta-test-bucket.3", BucketInfo{Name: "meta-test-bucket.3"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		// The err returned is of type *probe.Error.
+		bucketInfo, err := fs.GetBucketInfo(testCase.bucketName)
+
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.e.Error())
+
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if testCase.e.Error() != err.Cause.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.e.Error(), err.Cause.Error())
+			}
+		}
+		// Since there are cases for which GetBucketInfo fails, this is necessary.
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.bucketName != bucketInfo.Name {
+				t.Errorf("Test %d: Expected the bucket name to be \"%s\", but found \"%s\" instead", i+1, testCase.bucketName, bucketInfo.Name)
+			}
+		}
+	}
+}
+
+func TestListBuckets(t *testing.T) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a few buckets.
+	for i := 0; i < 10; i++ {
+		err = fs.MakeBucket("testbucket." + strconv.Itoa(i))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// List, and ensure that they are all there.
+	metadatas, err := fs.ListBuckets()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(metadatas) != 10 {
+		t.Errorf("incorrect length of metadatas (%d)\n", len(metadatas))
+	}
+
+	// Iterate over the buckets, ensuring that the name is correct.
+	for i := 0; i < len(metadatas); i++ {
+		if !strings.Contains(metadatas[i].Name, "testbucket") {
+			t.Fail()
+		}
+	}
+}
+
+func TestDeleteBucket(t *testing.T) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Deleting a bucket that doesn't exist should error.
+	err = fs.DeleteBucket("bucket")
+	if !strings.Contains(err.Cause.Error(), "Bucket not found:") {
+		t.Fail()
+	}
+}
+
+func BenchmarkListBuckets(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Create a few buckets.
+	for i := 0; i < 20; i++ {
+		err = fs.MakeBucket("bucket." + strconv.Itoa(i))
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+
+	// List the buckets over and over and over.
+	for i := 0; i < b.N; i++ {
+		_, err = fs.ListBuckets()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkDeleteBucket(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		// Creating buckets takes time, so stop and start the timer.
+		b.StopTimer()
+
+		// Create and delete the bucket over and over.
+		err = fs.MakeBucket("bucket")
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		b.StartTimer()
+
+		err = fs.DeleteBucket("bucket")
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkGetBucketInfo(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Put up a bucket with some metadata.
+	err = fs.MakeBucket("bucket")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		// Retrieve the metadata!
+		_, err := fs.GetBucketInfo("bucket")
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

fs-dir-common.go

@@ -0,0 +1,217 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+// fsDirent carries directory entries.
+type fsDirent struct {
+	name    string
+	modTime time.Time // On Solaris and older unix distros this is empty.
+	size    int64     // On Solaris and older unix distros this is empty.
+	mode    os.FileMode
+}
+
+// IsDir - returns true if fsDirent is a directory
+func (ent fsDirent) IsDir() bool {
+	return ent.mode.IsDir()
+}
+
+// IsSymlink - returns true if fsDirent is a symbolic link
+func (ent fsDirent) IsSymlink() bool {
+	return ent.mode&os.ModeSymlink == os.ModeSymlink
+}
+
+// IsRegular - returns true if fsDirent is a regular file
+func (ent fsDirent) IsRegular() bool {
+	return ent.mode.IsRegular()
+}
+
+// byDirentName is a collection satisfying sort.Interface.
+type byDirentName []fsDirent
+
+func (d byDirentName) Len() int           { return len(d) }
+func (d byDirentName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d byDirentName) Less(i, j int) bool { return d[i].name < d[j].name }
+
+// Using sort.Search() internally to jump to the file entry containing the prefix.
+func searchDirents(dirents []fsDirent, x string) int {
+	processFunc := func(i int) bool {
+		return dirents[i].name >= x
+	}
+	return sort.Search(len(dirents), processFunc)
+}
+
+// Tree walk result carries results of tree walking.
+type treeWalkResult struct {
+	objectInfo ObjectInfo
+	err        error
+	end        bool
+}
+
+// Tree walk notify carries a channel which notifies tree walk
+// results, additionally it also carries information if treeWalk
+// should be timedOut.
+type treeWalker struct {
+	ch       <-chan treeWalkResult
+	timedOut bool
+}
+
+// treeWalk walks FS directory tree recursively pushing ObjectInfo into the channel as and when it encounters files.
+func treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker string, recursive bool, send func(treeWalkResult) bool, count *int) bool {
+	// Example:
+	// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
+	// called with prefixDir="one/two/three/four/" and marker="five.txt"
+
+	// Convert dirent to ObjectInfo
+	direntToObjectInfo := func(dirent fsDirent) (ObjectInfo, error) {
+		objectInfo := ObjectInfo{}
+		// Convert to full object name.
+		objectInfo.Name = filepath.Join(prefixDir, dirent.name)
+		if dirent.modTime.IsZero() && dirent.size == 0 {
+			// ModifiedTime and Size are zero, Stat() and figure out
+			// the actual values that need to be set.
+			fi, err := os.Stat(filepath.Join(bucketDir, prefixDir, dirent.name))
+			if err != nil {
+				return ObjectInfo{}, err
+			}
+			// Fill size and modtime.
+			objectInfo.ModifiedTime = fi.ModTime()
+			objectInfo.Size = fi.Size()
+			objectInfo.IsDir = fi.IsDir()
+		} else {
+			// If ModifiedTime or Size are set then use them
+			// without attempting another Stat operation.
+			objectInfo.ModifiedTime = dirent.modTime
+			objectInfo.Size = dirent.size
+			objectInfo.IsDir = dirent.IsDir()
+		}
+		if objectInfo.IsDir {
+			// Add os.PathSeparator suffix again for directories as
+			// filepath.Join would have removed it.
+			objectInfo.Size = 0
+			objectInfo.Name += string(os.PathSeparator)
+		}
+		return objectInfo, nil
+	}
+
+	var markerBase, markerDir string
+	if marker != "" {
+		// Ex: if marker="four/five.txt", markerDir="four/" markerBase="five.txt"
+		markerSplit := strings.SplitN(marker, string(os.PathSeparator), 2)
+		markerDir = markerSplit[0]
+		if len(markerSplit) == 2 {
+			markerDir += string(os.PathSeparator)
+			markerBase = markerSplit[1]
+		}
+	}
+
+	// readDirAll returns entries that begins with entryPrefixMatch
+	dirents, err := readDirAll(filepath.Join(bucketDir, prefixDir), entryPrefixMatch)
+	if err != nil {
+		send(treeWalkResult{err: err})
+		return false
+	}
+	// example:
+	// If markerDir="four/" searchDirents() returns the index of "four/" in the sorted
+	// dirents list. We skip all the dirent entries till "four/"
+	dirents = dirents[searchDirents(dirents, markerDir):]
+	*count += len(dirents)
+	for i, dirent := range dirents {
+		if i == 0 && markerDir == dirent.name && !dirent.IsDir() {
+			// If the first entry is not a directory
+			// we need to skip this entry.
+			*count--
+			continue
+		}
+		if dirent.IsDir() && recursive {
+			// If the entry is a directory, we will need recurse into it.
+			markerArg := ""
+			if dirent.name == markerDir {
+				// We need to pass "five.txt" as marker only if we are
+				// recursing into "four/"
+				markerArg = markerBase
+			}
+			*count--
+			if !treeWalk(bucketDir, filepath.Join(prefixDir, dirent.name), "", markerArg, recursive, send, count) {
+				return false
+			}
+			continue
+		}
+		objectInfo, err := direntToObjectInfo(dirent)
+		if err != nil {
+			send(treeWalkResult{err: err})
+			return false
+		}
+		*count--
+		if !send(treeWalkResult{objectInfo: objectInfo}) {
+			return false
+		}
+	}
+	return true
+}
+
+// Initiate a new treeWalk in a goroutine.
+func startTreeWalk(fsPath, bucket, prefix, marker string, recursive bool) *treeWalker {
+	// Example 1
+	// If prefix is "one/two/three/" and marker is "one/two/three/four/five.txt"
+	// treeWalk is called with prefixDir="one/two/three/" and marker="four/five.txt"
+	// and entryPrefixMatch=""
+
+	// Example 2
+	// if prefix is "one/two/th" and marker is "one/two/three/four/five.txt"
+	// treeWalk is called with prefixDir="one/two/" and marker="three/four/five.txt"
+	// and entryPrefixMatch="th"
+	ch := make(chan treeWalkResult, listObjectsLimit)
+	walkNotify := treeWalker{ch: ch}
+	entryPrefixMatch := prefix
+	prefixDir := ""
+	lastIndex := strings.LastIndex(prefix, string(os.PathSeparator))
+	if lastIndex != -1 {
+		entryPrefixMatch = prefix[lastIndex+1:]
+		prefixDir = prefix[:lastIndex+1]
+	}
+	count := 0
+	marker = strings.TrimPrefix(marker, prefixDir)
+	go func() {
+		defer close(ch)
+		send := func(walkResult treeWalkResult) bool {
+			// Add the bucket.
+			walkResult.objectInfo.Bucket = bucket
+			if count == 0 {
+				walkResult.end = true
+			}
+			timer := time.After(time.Second * 60)
+			select {
+			case ch <- walkResult:
+				return true
+			case <-timer:
+				walkNotify.timedOut = true
+				return false
+			}
+		}
+		bucketDir := filepath.Join(fsPath, bucket)
+		treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker, recursive, send, &count)
+	}()
+	return &walkNotify
+}

fs-dir-nix.go

@@ -0,0 +1,170 @@
+// +build linux darwin dragonfly freebsd netbsd openbsd
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"syscall"
+	"unsafe"
+)
+
+const (
+	// readDirentBufSize for syscall.ReadDirent() to hold multiple
+	// directory entries in one buffer. golang source uses 4096 as
+	// buffer size whereas we want 25 times larger to save lots of
+	// entries to avoid multiple syscall.ReadDirent() call.
+	readDirentBufSize = 4096 * 25
+)
+
+// actual length of the byte array from the c - world.
+func clen(n []byte) int {
+	for i := 0; i < len(n); i++ {
+		if n[i] == 0 {
+			return i
+		}
+	}
+	return len(n)
+}
+
+// parseDirents - inspired from
+// https://golang.org/src/syscall/syscall_<os>.go
+func parseDirents(buf []byte) []fsDirent {
+	bufidx := 0
+	dirents := []fsDirent{}
+	for bufidx < len(buf) {
+		dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[bufidx]))
+		// On non-Linux operating systems for rec length of zero means
+		// we have reached EOF break out.
+		if runtime.GOOS != "linux" && dirent.Reclen == 0 {
+			break
+		}
+		bufidx += int(dirent.Reclen)
+		// Skip dirents if they are absent in directory.
+		if isEmptyDirent(dirent) {
+			continue
+		}
+		bytes := (*[10000]byte)(unsafe.Pointer(&dirent.Name[0]))
+		var name = string(bytes[0:clen(bytes[:])])
+		// Reserved names skip them.
+		if name == "." || name == ".." {
+			continue
+		}
+
+		var mode os.FileMode
+		switch dirent.Type {
+		case syscall.DT_BLK, syscall.DT_WHT:
+			mode = os.ModeDevice
+		case syscall.DT_CHR:
+			mode = os.ModeDevice | os.ModeCharDevice
+		case syscall.DT_DIR:
+			mode = os.ModeDir
+		case syscall.DT_FIFO:
+			mode = os.ModeNamedPipe
+		case syscall.DT_LNK:
+			mode = os.ModeSymlink
+		case syscall.DT_REG:
+			mode = 0
+		case syscall.DT_SOCK:
+			mode = os.ModeSocket
+		case syscall.DT_UNKNOWN:
+			mode = 0xffffffff
+		}
+
+		dirents = append(dirents, fsDirent{
+			name: name,
+			mode: mode,
+		})
+	}
+	return dirents
+}
+
+// Read all directory entries, returns a list of lexically sorted entries.
+func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) {
+	buf := make([]byte, readDirentBufSize)
+	f, err := os.Open(readDirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	dirents := []fsDirent{}
+	for {
+		nbuf, err := syscall.ReadDirent(int(f.Fd()), buf)
+		if err != nil {
+			return nil, err
+		}
+		if nbuf <= 0 {
+			break
+		}
+		for _, dirent := range parseDirents(buf[:nbuf]) {
+			if dirent.IsDir() {
+				dirent.name += string(os.PathSeparator)
+				dirent.size = 0
+			}
+			if strings.HasPrefix(dirent.name, entryPrefixMatch) {
+				dirents = append(dirents, dirent)
+			}
+		}
+	}
+	sort.Sort(byDirentName(dirents))
+	return dirents, nil
+}
+
+// scans the directory dirPath, calling filter() on each directory
+// entry.  Entries for which filter() returns true are stored, lexically
+// sorted using sort.Sort(). If filter is NULL, all entries are selected.
+// If namesOnly is true, dirPath is not appended into entry name.
+func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDirent, error) {
+	buf := make([]byte, readDirentBufSize)
+	d, err := os.Open(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer d.Close()
+
+	fd := int(d.Fd())
+	dirents := []fsDirent{}
+	for {
+		nbuf, err := syscall.ReadDirent(fd, buf)
+		if err != nil {
+			return nil, err
+		}
+		if nbuf <= 0 {
+			break
+		}
+		for _, dirent := range parseDirents(buf[:nbuf]) {
+			if !namesOnly {
+				dirent.name = filepath.Join(dirPath, dirent.name)
+			}
+			if dirent.IsDir() {
+				dirent.name += string(os.PathSeparator)
+			}
+			if filter == nil || filter(dirent) {
+				dirents = append(dirents, dirent)
+			}
+		}
+	}
+
+	sort.Sort(byDirentName(dirents))
+
+	return dirents, nil
+}

fs-dir-others.go

@@ -0,0 +1,108 @@
+// +build !linux,!darwin,!openbsd,!freebsd,!netbsd,!dragonfly
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// Read all directory entries, returns a list of lexically sorted entries.
+func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) {
+	f, err := os.Open(readDirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	var dirents []fsDirent
+	for {
+		fis, err := f.Readdir(1000)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		for _, fi := range fis {
+			dirent := fsDirent{
+				name:    fi.Name(),
+				modTime: fi.ModTime(),
+				size:    fi.Size(),
+				mode:    fi.Mode(),
+			}
+			if dirent.IsDir() {
+				dirent.name += string(os.PathSeparator)
+				dirent.size = 0
+			}
+			if strings.HasPrefix(fi.Name(), entryPrefixMatch) {
+				dirents = append(dirents, dirent)
+			}
+		}
+	}
+	// Sort dirents.
+	sort.Sort(byDirentName(dirents))
+	return dirents, nil
+}
+
+// scans the directory dirPath, calling filter() on each directory
+// entry.  Entries for which filter() returns true are stored, lexically
+// sorted using sort.Sort(). If filter is NULL, all entries are selected.
+// If namesOnly is true, dirPath is not appended into entry name.
+func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDirent, error) {
+	d, err := os.Open(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer d.Close()
+
+	var dirents []fsDirent
+	for {
+		fis, err := d.Readdir(1000)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		for _, fi := range fis {
+			dirent := fsDirent{
+				name:    fi.Name(),
+				modTime: fi.ModTime(),
+				size:    fi.Size(),
+				mode:    fi.Mode(),
+			}
+			if !namesOnly {
+				dirent.name = filepath.Join(dirPath, dirent.name)
+			}
+			if dirent.IsDir() {
+				dirent.name += string(os.PathSeparator)
+			}
+			if filter == nil || filter(dirent) {
+				dirents = append(dirents, dirent)
+			}
+		}
+	}
+
+	sort.Sort(byDirentName(dirents))
+
+	return dirents, nil
+}

fs-dirent-fileno.go

@@ -1,7 +1,7 @@
-// +build windows,amd64
+// +build openbsd netbsd freebsd dragonfly
 
 /*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,6 +16,11 @@
  * limitations under the License.
  */
 
-package sha1
+package main
 
-//go:generate yasm -f win64 -DWIN_ABI=1 sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso
+import "syscall"
+
+// True if dirent is absent in directory.
+func isEmptyDirent(dirent *syscall.Dirent) bool {
+	return dirent.Fileno == 0
+}

fs-dirent-ino.go

@@ -1,4 +1,4 @@
-// +build !cgo arm
+// +build darwin linux
 
 /*
  * Minio Cloud Storage, (C) 2016 Minio, Inc.
@@ -16,8 +16,11 @@
  * limitations under the License.
  */
 
-package sha1
+package main
 
-func block(dig *digest, p []byte) {
-	blockGeneric(dig, p)
+import "syscall"
+
+// True if dirent is absent in directory.
+func isEmptyDirent(dirent *syscall.Dirent) bool {
+	return dirent.Ino == 0
 }

fs-errors.go

@@ -14,17 +14,10 @@
  * limitations under the License.
  */
 
-package fs
+package main
 
 import "fmt"
 
-// SignDoesNotMatch - signature does not match.
-type SignDoesNotMatch struct{}
-
-func (e SignDoesNotMatch) Error() string {
-	return "Signature does not match."
-}
-
 // InvalidArgument invalid argument
 type InvalidArgument struct{}
 
@@ -131,15 +124,14 @@ func (e InvalidDisksArgument) Error() string {
 	return "Invalid number of disks per node"
 }
 
-// BadDigest bad md5sum
+// BadDigest - Content-MD5 you specified did not match what we received.
 type BadDigest struct {
-	MD5    string
-	Bucket string
-	Object string
+	ExpectedMD5   string
+	CalculatedMD5 string
 }
 
 func (e BadDigest) Error() string {
-	return "Bad digest"
+	return "Bad digest expected " + e.ExpectedMD5 + " is not valid with what we calculated " + e.CalculatedMD5
 }
 
 // InternalError - generic internal error
@@ -183,13 +175,6 @@ type ImplementationError struct {
 	Err    error
 }
 
-// DigestError - Generic MD5 error
-type DigestError struct {
-	Bucket string
-	Key    string
-	MD5    string
-}
-
 /// Bucket related errors
 
 // BucketNameInvalid - bucketname provided is invalid
@@ -200,9 +185,6 @@ type BucketNameInvalid GenericBucketError
 // ObjectNameInvalid - object name provided is invalid
 type ObjectNameInvalid GenericObjectError
 
-// InvalidDigest - md5 in request header invalid
-type InvalidDigest DigestError
-
 // Return string an error formatted as the given text
 func (e ImplementationError) Error() string {
 	error := ""
@@ -258,11 +240,6 @@ func (e BackendCorrupted) Error() string {
 	return "Backend corrupted: " + e.Path
 }
 
-// Return string an error formatted as the given text
-func (e InvalidDigest) Error() string {
-	return "MD5 provided " + e.MD5 + " is invalid"
-}
-
 // OperationNotPermitted - operation not permitted
 type OperationNotPermitted struct {
 	Op     string

fs-multipart-dir.go

@@ -0,0 +1,280 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func scanMultipartDir(bucketDir, prefixPath, markerPath, uploadIDMarker string, recursive bool) <-chan multipartObjectInfo {
+	objectInfoCh := make(chan multipartObjectInfo, listObjectsLimit)
+
+	// TODO: check if bucketDir is absolute path
+	scanDir := bucketDir
+	dirDepth := bucketDir
+
+	if prefixPath != "" {
+		if !filepath.IsAbs(prefixPath) {
+			tmpPrefixPath := filepath.Join(bucketDir, prefixPath)
+			if strings.HasSuffix(prefixPath, string(os.PathSeparator)) {
+				tmpPrefixPath += string(os.PathSeparator)
+			}
+			prefixPath = tmpPrefixPath
+		}
+
+		// TODO: check if prefixPath starts with bucketDir
+
+		// Case #1: if prefixPath is /mnt/mys3/mybucket/2012/photos/paris, then
+		//          dirDepth is /mnt/mys3/mybucket/2012/photos
+		// Case #2: if prefixPath is /mnt/mys3/mybucket/2012/photos/, then
+		//          dirDepth is /mnt/mys3/mybucket/2012/photos
+		dirDepth = filepath.Dir(prefixPath)
+		scanDir = dirDepth
+	} else {
+		prefixPath = bucketDir
+	}
+
+	if markerPath != "" {
+		if !filepath.IsAbs(markerPath) {
+			tmpMarkerPath := filepath.Join(bucketDir, markerPath)
+			if strings.HasSuffix(markerPath, string(os.PathSeparator)) {
+				tmpMarkerPath += string(os.PathSeparator)
+			}
+
+			markerPath = tmpMarkerPath
+		}
+
+		// TODO: check markerPath must be a file
+		if uploadIDMarker != "" {
+			markerPath = filepath.Join(markerPath, uploadIDMarker+multipartUploadIDSuffix)
+		}
+
+		// TODO: check if markerPath starts with bucketDir
+		// TODO: check if markerPath starts with prefixPath
+
+		// Case #1: if markerPath is /mnt/mys3/mybucket/2012/photos/gophercon.png, then
+		//          scanDir is /mnt/mys3/mybucket/2012/photos
+		// Case #2: if markerPath is /mnt/mys3/mybucket/2012/photos/gophercon.png/1fbd117a-268a-4ed0-85c9-8cc3888cbf20.uploadid, then
+		//          scanDir is /mnt/mys3/mybucket/2012/photos/gophercon.png
+		// Case #3: if markerPath is /mnt/mys3/mybucket/2012/photos/, then
+		//          scanDir is /mnt/mys3/mybucket/2012/photos
+
+		scanDir = filepath.Dir(markerPath)
+	} else {
+		markerPath = bucketDir
+	}
+
+	// Have bucketDir ends with os.PathSeparator
+	if !strings.HasSuffix(bucketDir, string(os.PathSeparator)) {
+		bucketDir += string(os.PathSeparator)
+	}
+
+	// Remove os.PathSeparator if scanDir ends with
+	if strings.HasSuffix(scanDir, string(os.PathSeparator)) {
+		scanDir = filepath.Dir(scanDir)
+	}
+
+	// goroutine - retrieves directory entries, makes ObjectInfo and sends into the channel.
+	go func() {
+		defer close(objectInfoCh)
+
+		// send function - returns true if ObjectInfo is sent
+		// within (time.Second * 15) else false on timeout.
+		send := func(oi multipartObjectInfo) bool {
+			timer := time.After(time.Second * 15)
+			select {
+			case objectInfoCh <- oi:
+				return true
+			case <-timer:
+				return false
+			}
+		}
+
+		// filter function - filters directory entries matching multipart uploadids, prefix and marker
+		direntFilterFn := func(dirent fsDirent) bool {
+			// check if dirent is a directory (or) dirent is a regular file and it's name ends with Upload ID suffix string
+			if dirent.IsDir() || (dirent.IsRegular() && strings.HasSuffix(dirent.name, multipartUploadIDSuffix)) {
+				// return if dirent's name starts with prefixPath and lexically higher than markerPath
+				return strings.HasPrefix(dirent.name, prefixPath) && dirent.name > markerPath
+			}
+			return false
+		}
+
+		// filter function - filters directory entries matching multipart uploadids
+		subDirentFilterFn := func(dirent fsDirent) bool {
+			// check if dirent is a directory (or) dirent is a regular file and it's name ends with Upload ID suffix string
+			return dirent.IsDir() || (dirent.IsRegular() && strings.HasSuffix(dirent.name, multipartUploadIDSuffix))
+		}
+
+		// lastObjInfo is used to save last object info which is sent at last with End=true
+		var lastObjInfo *multipartObjectInfo
+
+		sendError := func(err error) {
+			if lastObjInfo != nil {
+				if !send(*lastObjInfo) {
+					// as we got error sending lastObjInfo, we can't send the error
+					return
+				}
+			}
+
+			send(multipartObjectInfo{Err: err, End: true})
+			return
+		}
+
+		for {
+			dirents, err := scandir(scanDir, direntFilterFn, false)
+			if err != nil {
+				sendError(err)
+				return
+			}
+
+			var dirent fsDirent
+			for len(dirents) > 0 {
+				dirent, dirents = dirents[0], dirents[1:]
+				if dirent.IsRegular() {
+					// Handle uploadid file
+					name := strings.Replace(filepath.Dir(dirent.name), bucketDir, "", 1)
+					if name == "" {
+						// This should not happen ie uploadid file should not be in bucket directory
+						sendError(errors.New("Corrupted metadata"))
+						return
+					}
+
+					uploadID := strings.Split(filepath.Base(dirent.name), multipartUploadIDSuffix)[0]
+
+					// Solaris and older unixes have modTime to be
+					// empty, fallback to os.Stat() to fill missing values.
+					if dirent.modTime.IsZero() {
+						if fi, e := os.Stat(dirent.name); e == nil {
+							dirent.modTime = fi.ModTime()
+						} else {
+							sendError(e)
+							return
+						}
+					}
+
+					objInfo := multipartObjectInfo{
+						Name:         name,
+						UploadID:     uploadID,
+						ModifiedTime: dirent.modTime,
+					}
+
+					// as we got new object info, send last object info and keep new object info as last object info
+					if lastObjInfo != nil {
+						if !send(*lastObjInfo) {
+							return
+						}
+					}
+					lastObjInfo = &objInfo
+
+					continue
+				}
+
+				// Fetch sub dirents.
+				subDirents, err := scandir(dirent.name, subDirentFilterFn, false)
+				if err != nil {
+					sendError(err)
+					return
+				}
+
+				subDirFound := false
+				uploadIDDirents := []fsDirent{}
+				// If subDirents has a directory, then current dirent needs to be sent
+				for _, subdirent := range subDirents {
+					if subdirent.IsDir() {
+						subDirFound = true
+
+						if recursive {
+							break
+						}
+					}
+
+					if !recursive && subdirent.IsRegular() {
+						uploadIDDirents = append(uploadIDDirents, subdirent)
+					}
+				}
+
+				// Send directory only for non-recursive listing
+				if !recursive && (subDirFound || len(subDirents) == 0) {
+					// Solaris and older unixes have modTime to be
+					// empty, fallback to os.Stat() to fill missing values.
+					if dirent.modTime.IsZero() {
+						if fi, e := os.Stat(dirent.name); e == nil {
+							dirent.modTime = fi.ModTime()
+						} else {
+							sendError(e)
+							return
+						}
+					}
+
+					objInfo := multipartObjectInfo{
+						Name:         strings.Replace(dirent.name, bucketDir, "", 1),
+						ModifiedTime: dirent.modTime,
+						IsDir:        true,
+					}
+
+					// as we got new object info, send last object info and keep new object info as last object info
+					if lastObjInfo != nil {
+						if !send(*lastObjInfo) {
+							return
+						}
+					}
+					lastObjInfo = &objInfo
+				}
+
+				if recursive {
+					dirents = append(subDirents, dirents...)
+				} else {
+					dirents = append(uploadIDDirents, dirents...)
+				}
+			}
+
+			if !recursive {
+				break
+			}
+
+			markerPath = scanDir + string(os.PathSeparator)
+			if scanDir = filepath.Dir(scanDir); scanDir < dirDepth {
+				break
+			}
+		}
+
+		if lastObjInfo != nil {
+			// we got last object
+			lastObjInfo.End = true
+			if !send(*lastObjInfo) {
+				return
+			}
+		}
+	}()
+
+	return objectInfoCh
+}
+
+// multipartObjectInfo - Multipart object info
+type multipartObjectInfo struct {
+	Name         string
+	UploadID     string
+	ModifiedTime time.Time
+	IsDir        bool
+	Err          error
+	End          bool
+}

fs-multipart.go

@@ -0,0 +1,685 @@
+/*
+ * Minio Cloud Storage, (C) 2015,2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/pkg/mimedb"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/safe"
+	"github.com/skyrings/skyring-common/tools/uuid"
+)
+
+const (
+	minioMetaDir            = ".minio"
+	multipartUploadIDSuffix = ".uploadid"
+)
+
+// Removes files and its parent directories up to a given level.
+func removeFileTree(fileName string, level string) error {
+	if e := os.Remove(fileName); e != nil {
+		return e
+	}
+
+	for fileDir := filepath.Dir(fileName); fileDir > level; fileDir = filepath.Dir(fileDir) {
+		if status, e := isDirEmpty(fileDir); e != nil {
+			return e
+		} else if !status {
+			break
+		}
+		if e := os.Remove(fileDir); e != nil {
+			return e
+		}
+	}
+
+	return nil
+}
+
+// Takes an input stream and safely writes to disk, additionally
+// verifies checksum.
+func safeWriteFile(fileName string, data io.Reader, size int64, md5sum string) error {
+	safeFile, e := safe.CreateFileWithSuffix(fileName, "-")
+	if e != nil {
+		return e
+	}
+
+	md5Hasher := md5.New()
+	multiWriter := io.MultiWriter(md5Hasher, safeFile)
+	if size > 0 {
+		if _, e = io.CopyN(multiWriter, data, size); e != nil {
+			// Closes the file safely and removes it in a single atomic operation.
+			safeFile.CloseAndRemove()
+			return e
+		}
+	} else {
+		if _, e = io.Copy(multiWriter, data); e != nil {
+			// Closes the file safely and removes it in a single atomic operation.
+			safeFile.CloseAndRemove()
+			return e
+		}
+	}
+
+	dataMd5sum := hex.EncodeToString(md5Hasher.Sum(nil))
+	if md5sum != "" && !isMD5SumEqual(md5sum, dataMd5sum) {
+		// Closes the file safely and removes it in a single atomic operation.
+		safeFile.CloseAndRemove()
+		return BadDigest{ExpectedMD5: md5sum, CalculatedMD5: dataMd5sum}
+	}
+
+	// Safely close the file and atomically renames it the actual filePath.
+	safeFile.Close()
+
+	// Safely wrote the file.
+	return nil
+}
+
+func isFileExist(filename string) (bool, error) {
+	fi, e := os.Lstat(filename)
+	if e != nil {
+		if os.IsNotExist(e) {
+			return false, nil
+		}
+
+		return false, e
+	}
+
+	return fi.Mode().IsRegular(), nil
+}
+
+// Create an s3 compatible MD5sum for complete multipart transaction.
+func makeS3MD5(md5Strs ...string) (string, *probe.Error) {
+	var finalMD5Bytes []byte
+	for _, md5Str := range md5Strs {
+		md5Bytes, e := hex.DecodeString(md5Str)
+		if e != nil {
+			return "", probe.NewError(e)
+		}
+		finalMD5Bytes = append(finalMD5Bytes, md5Bytes...)
+	}
+	md5Hasher := md5.New()
+	md5Hasher.Write(finalMD5Bytes)
+	s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(md5Strs))
+	return s3MD5, nil
+}
+
+func (fs Filesystem) newUploadID(bucket, object string) (string, error) {
+	metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
+
+	// create metaObjectDir if not exist
+	if status, e := isDirExist(metaObjectDir); e != nil {
+		return "", e
+	} else if !status {
+		if e := os.MkdirAll(metaObjectDir, 0755); e != nil {
+			return "", e
+		}
+	}
+
+	for {
+		uuid, e := uuid.New()
+		if e != nil {
+			return "", e
+		}
+
+		uploadID := uuid.String()
+		uploadIDFile := filepath.Join(metaObjectDir, uploadID+multipartUploadIDSuffix)
+		if _, e := os.Lstat(uploadIDFile); e != nil {
+			if !os.IsNotExist(e) {
+				return "", e
+			}
+
+			// uploadIDFile doesn't exist, so create empty file to reserve the name
+			if e := ioutil.WriteFile(uploadIDFile, []byte{}, 0644); e != nil {
+				return "", e
+			}
+
+			return uploadID, nil
+		}
+		// uploadIDFile already exists.
+		// loop again to try with different uuid generated.
+	}
+}
+
+func (fs Filesystem) isUploadIDExist(bucket, object, uploadID string) (bool, error) {
+	return isFileExist(filepath.Join(fs.diskPath, minioMetaDir, bucket, object, uploadID+multipartUploadIDSuffix))
+}
+
+func (fs Filesystem) cleanupUploadID(bucket, object, uploadID string) error {
+	metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
+	uploadIDPrefix := uploadID + "."
+
+	dirents, e := scandir(metaObjectDir,
+		func(dirent fsDirent) bool {
+			return dirent.IsRegular() && strings.HasPrefix(dirent.name, uploadIDPrefix)
+		},
+		true)
+
+	if e != nil {
+		return e
+	}
+
+	for _, dirent := range dirents {
+		if e := os.Remove(filepath.Join(metaObjectDir, dirent.name)); e != nil {
+			return e
+		}
+	}
+
+	if status, e := isDirEmpty(metaObjectDir); e != nil {
+		return e
+	} else if status {
+		if e := removeFileTree(metaObjectDir, filepath.Join(fs.diskPath, minioMetaDir, bucket)); e != nil {
+			return e
+		}
+	}
+
+	return nil
+}
+
+func (fs Filesystem) checkMultipartArgs(bucket, object string) (string, error) {
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return "", e
+	}
+
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{Object: object}
+	}
+
+	return bucket, nil
+}
+
+// NewMultipartUpload - initiate a new multipart session
+func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.Error) {
+	if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
+		bucket = bucketDirName
+	} else {
+		return "", probe.NewError(e)
+	}
+
+	if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
+		return "", probe.NewError(e)
+	}
+
+	uploadID, e := fs.newUploadID(bucket, object)
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+
+	return uploadID, nil
+}
+
+// PutObjectPart - create a part in a multipart session
+func (fs Filesystem) PutObjectPart(bucket, object, uploadID string, partNumber int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) {
+	if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
+		bucket = bucketDirName
+	} else {
+		return "", probe.NewError(e)
+	}
+
+	if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil {
+		//return "", probe.NewError(InternalError{Err: err})
+		return "", probe.NewError(e)
+	} else if !status {
+		return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	// Part id cannot be negative.
+	if partNumber <= 0 {
+		return "", probe.NewError(errors.New("invalid part id, cannot be zero or less than zero"))
+	}
+
+	if partNumber > 10000 {
+		return "", probe.NewError(errors.New("invalid part id, should be not more than 10000"))
+	}
+
+	if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
+		return "", probe.NewError(e)
+	}
+
+	partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partNumber, md5Hex)
+	partFilePath := filepath.Join(fs.diskPath, minioMetaDir, bucket, object, partSuffix)
+	if e := safeWriteFile(partFilePath, data, size, md5Hex); e != nil {
+		return "", probe.NewError(e)
+	}
+	return md5Hex, nil
+}
+
+// AbortMultipartUpload - abort an incomplete multipart session
+func (fs Filesystem) AbortMultipartUpload(bucket, object, uploadID string) *probe.Error {
+	if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
+		bucket = bucketDirName
+	} else {
+		return probe.NewError(e)
+	}
+
+	if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil {
+		//return probe.NewError(InternalError{Err: err})
+		return probe.NewError(e)
+	} else if !status {
+		return probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	if e := fs.cleanupUploadID(bucket, object, uploadID); e != nil {
+		return probe.NewError(e)
+	}
+
+	return nil
+}
+
+// CompleteMultipartUpload - complete a multipart upload and persist the data
+func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, parts []completePart) (ObjectInfo, *probe.Error) {
+	if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
+		bucket = bucketDirName
+	} else {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil {
+		//return probe.NewError(InternalError{Err: err})
+		return ObjectInfo{}, probe.NewError(e)
+	} else if !status {
+		return ObjectInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
+
+	var md5Sums []string
+	for _, part := range parts {
+		partNumber := part.PartNumber
+		md5sum := strings.Trim(part.ETag, "\"")
+		partFile := filepath.Join(metaObjectDir, uploadID+"."+strconv.Itoa(partNumber)+"."+md5sum)
+		if status, err := isFileExist(partFile); err != nil {
+			return ObjectInfo{}, probe.NewError(err)
+		} else if !status {
+			return ObjectInfo{}, probe.NewError(InvalidPart{})
+		}
+		md5Sums = append(md5Sums, md5sum)
+	}
+
+	// Save the s3 md5.
+	s3MD5, err := makeS3MD5(md5Sums...)
+	if err != nil {
+		return ObjectInfo{}, err.Trace(md5Sums...)
+	}
+
+	completeObjectFile := filepath.Join(metaObjectDir, uploadID+".complete.")
+	safeFile, e := safe.CreateFileWithSuffix(completeObjectFile, "-")
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+	for _, part := range parts {
+		partNumber := part.PartNumber
+		// Trim off the odd double quotes from ETag in the beginning and end.
+		md5sum := strings.TrimPrefix(part.ETag, "\"")
+		md5sum = strings.TrimSuffix(md5sum, "\"")
+		partFileStr := filepath.Join(metaObjectDir, fmt.Sprintf("%s.%d.%s", uploadID, partNumber, md5sum))
+		var partFile *os.File
+		partFile, e = os.Open(partFileStr)
+		if e != nil {
+			// Remove the complete file safely.
+			safeFile.CloseAndRemove()
+			return ObjectInfo{}, probe.NewError(e)
+		} else if _, e = io.Copy(safeFile, partFile); e != nil {
+			// Remove the complete file safely.
+			safeFile.CloseAndRemove()
+			return ObjectInfo{}, probe.NewError(e)
+		}
+		partFile.Close() // Close part file after successful copy.
+	}
+	// All parts concatenated, safely close the temp file.
+	safeFile.Close()
+
+	// Stat to gather fresh stat info.
+	objSt, e := os.Stat(completeObjectFile)
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	bucketPath := filepath.Join(fs.diskPath, bucket)
+	objectPath := filepath.Join(bucketPath, object)
+	if e = os.MkdirAll(filepath.Dir(objectPath), 0755); e != nil {
+		os.Remove(completeObjectFile)
+		return ObjectInfo{}, probe.NewError(e)
+	}
+	if e = os.Rename(completeObjectFile, objectPath); e != nil {
+		os.Remove(completeObjectFile)
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	fs.cleanupUploadID(bucket, object, uploadID) // TODO: handle and log the error
+
+	contentType := "application/octet-stream"
+	if objectExt := filepath.Ext(objectPath); objectExt != "" {
+		if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
+			contentType = content.ContentType
+		}
+	}
+
+	newObject := ObjectInfo{
+		Bucket:       bucket,
+		Name:         object,
+		ModifiedTime: objSt.ModTime(),
+		Size:         objSt.Size(),
+		ContentType:  contentType,
+		MD5Sum:       s3MD5,
+	}
+
+	return newObject, nil
+}
+
+func (fs *Filesystem) saveListMultipartObjectCh(params listMultipartObjectParams, ch <-chan multipartObjectInfo) {
+	fs.listMultipartObjectMapMutex.Lock()
+	defer fs.listMultipartObjectMapMutex.Unlock()
+
+	channels := []<-chan multipartObjectInfo{ch}
+	if _, ok := fs.listMultipartObjectMap[params]; ok {
+		channels = append(fs.listMultipartObjectMap[params], ch)
+	}
+
+	fs.listMultipartObjectMap[params] = channels
+}
+
+func (fs *Filesystem) lookupListMultipartObjectCh(params listMultipartObjectParams) <-chan multipartObjectInfo {
+	fs.listMultipartObjectMapMutex.Lock()
+	defer fs.listMultipartObjectMapMutex.Unlock()
+
+	if channels, ok := fs.listMultipartObjectMap[params]; ok {
+		var channel <-chan multipartObjectInfo
+		channel, channels = channels[0], channels[1:]
+		if len(channels) > 0 {
+			fs.listMultipartObjectMap[params] = channels
+		} else {
+			// do not store empty channel list
+			delete(fs.listMultipartObjectMap, params)
+		}
+
+		return channel
+	}
+
+	return nil
+}
+
+// ListMultipartUploads - list incomplete multipart sessions for a given BucketMultipartResourcesMetadata
+func (fs Filesystem) ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error) {
+	result := ListMultipartsInfo{}
+
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return result, probe.NewError(e)
+	}
+
+	if !IsValidObjectPrefix(objectPrefix) {
+		return result, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: objectPrefix})
+	}
+
+	prefixPath := filepath.FromSlash(objectPrefix)
+
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported", delimiter))
+	}
+
+	if keyMarker != "" && !strings.HasPrefix(keyMarker, objectPrefix) {
+		return result, probe.NewError(fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", keyMarker, objectPrefix))
+	}
+
+	markerPath := filepath.FromSlash(keyMarker)
+	if uploadIDMarker != "" {
+		if strings.HasSuffix(markerPath, string(os.PathSeparator)) {
+			return result, probe.NewError(fmt.Errorf("Invalid combination of uploadID marker '%s' and marker '%s'", uploadIDMarker, keyMarker))
+		}
+		id, e := uuid.Parse(uploadIDMarker)
+		if e != nil {
+			return result, probe.NewError(e)
+		}
+		if id.IsZero() {
+			return result, probe.NewError(fmt.Errorf("Invalid upload ID marker %s", uploadIDMarker))
+		}
+	}
+
+	// Return empty response if maxUploads is zero
+	if maxUploads == 0 {
+		return result, nil
+	}
+
+	// set listObjectsLimit to maxUploads for out-of-range limit
+	if maxUploads < 0 || maxUploads > listObjectsLimit {
+		maxUploads = listObjectsLimit
+	}
+
+	recursive := true
+	if delimiter == "/" {
+		recursive = false
+	}
+
+	metaBucketDir := filepath.Join(fs.diskPath, minioMetaDir, bucket)
+
+	// Lookup of if listMultipartObjectChannel is available for given
+	// parameters, else create a new one.
+	savedChannel := true
+	multipartObjectInfoCh := fs.lookupListMultipartObjectCh(listMultipartObjectParams{
+		bucket:         bucket,
+		delimiter:      delimiter,
+		keyMarker:      markerPath,
+		prefix:         prefixPath,
+		uploadIDMarker: uploadIDMarker,
+	})
+
+	if multipartObjectInfoCh == nil {
+		multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, keyMarker, uploadIDMarker, recursive)
+		savedChannel = false
+	}
+
+	var objInfo *multipartObjectInfo
+	nextKeyMarker := ""
+	nextUploadIDMarker := ""
+	for i := 0; i < maxUploads; {
+		// read the channel
+		if oi, ok := <-multipartObjectInfoCh; ok {
+			objInfo = &oi
+		} else {
+			// closed channel
+			if i == 0 {
+				// first read
+				if !savedChannel {
+					// its valid to have a closed new channel for first read
+					multipartObjectInfoCh = nil
+					break
+				}
+
+				// invalid saved channel amd create new channel
+				multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, keyMarker,
+					uploadIDMarker, recursive)
+			} else {
+				// TODO: FIX: there is a chance of infinite loop if we get closed channel always
+				// the channel got closed due to timeout
+				// create a new channel
+				multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, nextKeyMarker,
+					nextUploadIDMarker, recursive)
+			}
+
+			// make it as new channel
+			savedChannel = false
+			continue
+		}
+
+		if objInfo.Err != nil {
+			if os.IsNotExist(objInfo.Err) {
+				return ListMultipartsInfo{}, nil
+			}
+
+			return ListMultipartsInfo{}, probe.NewError(objInfo.Err)
+		}
+
+		// backward compatibility check
+		if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") {
+			continue
+		}
+
+		// Directories are listed only if recursive is false
+		if objInfo.IsDir {
+			result.CommonPrefixes = append(result.CommonPrefixes, objInfo.Name)
+		} else {
+			result.Uploads = append(result.Uploads, uploadMetadata{
+				Object:    objInfo.Name,
+				UploadID:  objInfo.UploadID,
+				Initiated: objInfo.ModifiedTime,
+			})
+		}
+
+		nextKeyMarker = objInfo.Name
+		nextUploadIDMarker = objInfo.UploadID
+		i++
+
+		if objInfo.End {
+			// as we received last object, do not save this channel for later use
+			multipartObjectInfoCh = nil
+			break
+		}
+	}
+
+	if multipartObjectInfoCh != nil {
+		// we haven't received last object
+		result.IsTruncated = true
+		result.NextKeyMarker = nextKeyMarker
+		result.NextUploadIDMarker = nextUploadIDMarker
+
+		// save this channel for later use
+		fs.saveListMultipartObjectCh(listMultipartObjectParams{
+			bucket:         bucket,
+			delimiter:      delimiter,
+			keyMarker:      nextKeyMarker,
+			prefix:         objectPrefix,
+			uploadIDMarker: nextUploadIDMarker,
+		}, multipartObjectInfoCh)
+	}
+
+	return result, nil
+}
+
+// ListObjectParts - list parts from incomplete multipart session for a given ObjectResourcesMetadata
+func (fs Filesystem) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error) {
+	if bucketDirName, err := fs.checkMultipartArgs(bucket, object); err == nil {
+		bucket = bucketDirName
+	} else {
+		return ListPartsInfo{}, probe.NewError(err)
+	}
+
+	if status, err := fs.isUploadIDExist(bucket, object, uploadID); err != nil {
+		//return probe.NewError(InternalError{Err: err})
+		return ListPartsInfo{}, probe.NewError(err)
+	} else if !status {
+		return ListPartsInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	// return empty ListPartsInfo
+	if maxParts == 0 {
+		return ListPartsInfo{}, nil
+	}
+
+	if maxParts < 0 || maxParts > 1000 {
+		maxParts = 1000
+	}
+
+	metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
+	uploadIDPrefix := uploadID + "."
+
+	dirents, e := scandir(metaObjectDir,
+		func(dirent fsDirent) bool {
+			// Part file is a regular file and has to be started with 'UPLOADID.'
+			if !(dirent.IsRegular() && strings.HasPrefix(dirent.name, uploadIDPrefix)) {
+				return false
+			}
+
+			// Valid part file has to be 'UPLOADID.PARTNUMBER.MD5SUM'
+			tokens := strings.Split(dirent.name, ".")
+			if len(tokens) != 3 {
+				return false
+			}
+
+			if partNumber, err := strconv.Atoi(tokens[1]); err == nil {
+				if partNumber >= 1 && partNumber <= 10000 && partNumber > partNumberMarker {
+					return true
+				}
+			}
+
+			return false
+		},
+		true)
+	if e != nil {
+		return ListPartsInfo{}, probe.NewError(e)
+	}
+
+	isTruncated := false
+	nextPartNumberMarker := 0
+
+	parts := []partInfo{}
+	for i := range dirents {
+		if i == maxParts {
+			isTruncated = true
+			break
+		}
+
+		// In some OS modTime is empty and use os.Stat() to fill missing values
+		if dirents[i].modTime.IsZero() {
+			if fi, e := os.Stat(filepath.Join(metaObjectDir, dirents[i].name)); e == nil {
+				dirents[i].modTime = fi.ModTime()
+				dirents[i].size = fi.Size()
+			} else {
+				return ListPartsInfo{}, probe.NewError(e)
+			}
+		}
+
+		tokens := strings.Split(dirents[i].name, ".")
+		partNumber, _ := strconv.Atoi(tokens[1])
+		md5sum := tokens[2]
+		parts = append(parts, partInfo{
+			PartNumber:   partNumber,
+			LastModified: dirents[i].modTime,
+			ETag:         md5sum,
+			Size:         dirents[i].size,
+		})
+	}
+
+	if isTruncated {
+		nextPartNumberMarker = 0
+	}
+
+	return ListPartsInfo{
+		Bucket:               bucket,
+		Object:               object,
+		UploadID:             uploadID,
+		PartNumberMarker:     partNumberMarker,
+		NextPartNumberMarker: nextPartNumberMarker,
+		MaxParts:             maxParts,
+		IsTruncated:          isTruncated,
+		Parts:                parts,
+	}, nil
+}

fs-object.go

@@ -0,0 +1,338 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"encoding/hex"
+	"runtime"
+
+	"github.com/minio/minio/pkg/mimedb"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/safe"
+)
+
+/// Object Operations
+
+// GetObject - GET object
+func (fs Filesystem) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) {
+	// Input validation.
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	if !IsValidObjectName(object) {
+		return nil, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	objectPath := filepath.Join(fs.diskPath, bucket, object)
+	file, e := os.Open(objectPath)
+	if e != nil {
+		// If the object doesn't exist, the bucket might not exist either. Stat for
+		// the bucket and give a better error message if that is true.
+		if os.IsNotExist(e) {
+			_, e = os.Stat(filepath.Join(fs.diskPath, bucket))
+			if os.IsNotExist(e) {
+				return nil, probe.NewError(BucketNotFound{Bucket: bucket})
+			}
+			return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return nil, probe.NewError(e)
+	}
+	// Initiate a cached stat operation on the file handler.
+	st, e := file.Stat()
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	// Object path is a directory prefix, return object not found error.
+	if st.IsDir() {
+		return nil, probe.NewError(ObjectExistsAsPrefix{
+			Bucket: bucket,
+			Prefix: object,
+		})
+	}
+
+	// Seek to a starting offset.
+	_, e = file.Seek(startOffset, os.SEEK_SET)
+	if e != nil {
+		// When the "handle is invalid", the file might be a directory on Windows.
+		if runtime.GOOS == "windows" && strings.Contains(e.Error(), "handle is invalid") {
+			return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return nil, probe.NewError(e)
+	}
+	return file, nil
+}
+
+// GetObjectInfo - get object info.
+func (fs Filesystem) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) {
+	// Input validation.
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	if !IsValidObjectName(object) {
+		return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	info, err := getObjectInfo(fs.diskPath, bucket, object)
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return ObjectInfo{}, err.Trace(bucket, object)
+	}
+	if info.IsDir {
+		return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+	}
+	return info, nil
+}
+
+// getObjectInfo - get object stat info.
+func getObjectInfo(rootPath, bucket, object string) (ObjectInfo, *probe.Error) {
+	// Do not use filepath.Join() since filepath.Join strips off any
+	// object names with '/', use them as is in a static manner so
+	// that we can send a proper 'ObjectNotFound' reply back upon os.Stat().
+	var objectPath string
+	// For windows use its special os.PathSeparator == "\\"
+	if runtime.GOOS == "windows" {
+		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	} else {
+		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	}
+	stat, e := os.Stat(objectPath)
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+	contentType := "application/octet-stream"
+	if runtime.GOOS == "windows" {
+		object = filepath.ToSlash(object)
+	}
+
+	if objectExt := filepath.Ext(object); objectExt != "" {
+		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
+		if ok {
+			contentType = content.ContentType
+		}
+	}
+	objInfo := ObjectInfo{
+		Bucket:       bucket,
+		Name:         object,
+		ModifiedTime: stat.ModTime(),
+		Size:         stat.Size(),
+		ContentType:  contentType,
+		IsDir:        stat.Mode().IsDir(),
+	}
+	return objInfo, nil
+}
+
+// isMD5SumEqual - returns error if md5sum mismatches, success its `nil`
+func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
+	// Verify the md5sum.
+	if expectedMD5Sum != "" && actualMD5Sum != "" {
+		// Decode md5sum to bytes from their hexadecimal
+		// representations.
+		expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum)
+		if err != nil {
+			return false
+		}
+		actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum)
+		if err != nil {
+			return false
+		}
+		// Verify md5sum bytes are equal after successful decoding.
+		if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+// PutObject - create an object.
+func (fs Filesystem) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error) {
+	// Check bucket name valid.
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	bucketDir := filepath.Join(fs.diskPath, bucket)
+
+	// Verify object path legal.
+	if !IsValidObjectName(object) {
+		return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	if e = checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	// Get object path.
+	objectPath := filepath.Join(bucketDir, object)
+
+	// md5Hex representation.
+	var md5Hex string
+	if len(metadata) != 0 {
+		md5Hex = metadata["md5Sum"]
+	}
+
+	// Write object.
+	safeFile, e := safe.CreateFileWithPrefix(objectPath, md5Hex+"$tmpobject")
+	if e != nil {
+		switch e := e.(type) {
+		case *os.PathError:
+			if e.Op == "mkdir" {
+				if strings.Contains(e.Error(), "not a directory") {
+					return ObjectInfo{}, probe.NewError(ObjectExistsAsPrefix{Bucket: bucket, Prefix: object})
+				}
+			}
+			return ObjectInfo{}, probe.NewError(e)
+		default:
+			return ObjectInfo{}, probe.NewError(e)
+		}
+	}
+
+	// Initialize md5 writer.
+	md5Writer := md5.New()
+
+	// Instantiate a new multi writer.
+	multiWriter := io.MultiWriter(md5Writer, safeFile)
+
+	// Instantiate checksum hashers and create a multiwriter.
+	if size > 0 {
+		if _, e = io.CopyN(multiWriter, data, size); e != nil {
+			safeFile.CloseAndRemove()
+			return ObjectInfo{}, probe.NewError(e)
+		}
+	} else {
+		if _, e = io.Copy(multiWriter, data); e != nil {
+			safeFile.CloseAndRemove()
+			return ObjectInfo{}, probe.NewError(e)
+		}
+	}
+
+	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
+	if md5Hex != "" {
+		if newMD5Hex != md5Hex {
+			return ObjectInfo{}, probe.NewError(BadDigest{md5Hex, newMD5Hex})
+		}
+	}
+
+	// Set stat again to get the latest metadata.
+	st, e := os.Stat(safeFile.Name())
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	contentType := "application/octet-stream"
+	if objectExt := filepath.Ext(objectPath); objectExt != "" {
+		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
+		if ok {
+			contentType = content.ContentType
+		}
+	}
+	newObject := ObjectInfo{
+		Bucket:       bucket,
+		Name:         object,
+		ModifiedTime: st.ModTime(),
+		Size:         st.Size(),
+		MD5Sum:       newMD5Hex,
+		ContentType:  contentType,
+	}
+
+	// Safely close and atomically rename the file.
+	safeFile.Close()
+
+	return newObject, nil
+}
+
+// deleteObjectPath - delete object path if its empty.
+func deleteObjectPath(basePath, deletePath, bucket, object string) *probe.Error {
+	if basePath == deletePath {
+		return nil
+	}
+	// Verify if the path exists.
+	pathSt, e := os.Stat(deletePath)
+	if e != nil {
+		if os.IsNotExist(e) {
+			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return probe.NewError(e)
+	}
+	if pathSt.IsDir() {
+		// Verify if directory is empty.
+		empty, e := isDirEmpty(deletePath)
+		if e != nil {
+			return probe.NewError(e)
+		}
+		if !empty {
+			return nil
+		}
+	}
+	// Attempt to remove path.
+	if e := os.Remove(deletePath); e != nil {
+		return probe.NewError(e)
+	}
+	// Recursively go down the next path and delete again.
+	if err := deleteObjectPath(basePath, filepath.Dir(deletePath), bucket, object); err != nil {
+		return err.Trace(basePath, deletePath, bucket, object)
+	}
+	return nil
+}
+
+// DeleteObject - delete object.
+func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error {
+	// Check bucket name valid
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return probe.NewError(e)
+	}
+
+	bucketDir := filepath.Join(fs.diskPath, bucket)
+
+	// Verify object path legal
+	if !IsValidObjectName(object) {
+		return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	// Do not use filepath.Join() since filepath.Join strips off any
+	// object names with '/', use them as is in a static manner so
+	// that we can send a proper 'ObjectNotFound' reply back upon os.Stat().
+	var objectPath string
+	if runtime.GOOS == "windows" {
+		objectPath = fs.diskPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	} else {
+		objectPath = fs.diskPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	}
+	// Delete object path if its empty.
+	err := deleteObjectPath(bucketDir, objectPath, bucket, object)
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return err.Trace(bucketDir, objectPath, bucket, object)
+	}
+	return nil
+}

fs-object_test.go

@@ -0,0 +1,255 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Testing GetObjectInfo().
+func TestGetObjectInfo(t *testing.T) {
+	directory, e := ioutil.TempDir("", "minio-get-objinfo-test")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// This bucket is used for testing getObjectInfo operations.
+	err = fs.MakeBucket("test-getobjectinfo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fs.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resultCases := []ObjectInfo{
+		// ObjectInfo -1.
+		// ObjectName set to a existing object in the test case (Test case 14).
+		{Bucket: "test-getobjectinfo", Name: "Asia/asiapics.jpg", ContentType: "image/jpeg", IsDir: false},
+	}
+	testCases := []struct {
+		bucketName string
+		objectName string
+
+		// Expected output of GetObjectInfo.
+		result ObjectInfo
+		err    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names ( Test number 1-4 ).
+		{".test", "", ObjectInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Test cases with valid but non-existing bucket names (Test number 5-7).
+		{"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false},
+		{"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false},
+		// Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9).
+		{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
+		{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
+		// Test cases with non-existing object name with existing bucket (Test number 10-12).
+		{"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false},
+		{"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false},
+		{"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false},
+		// Test case with existing bucket but object name set to a directory (Test number 13).
+		{"test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false},
+		// Valid case with existing object (Test number 14).
+		{"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true},
+	}
+	for i, testCase := range testCases {
+		result, err := fs.GetObjectInfo(testCase.bucketName, testCase.objectName)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if testCase.err.Error() != err.Cause.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error())
+			}
+		}
+
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.result.Bucket != result.Bucket {
+				t.Fatalf("Test %d: Expected Bucket name to be '%s', but found '%s' instead", i+1, testCase.result.Bucket, result.Bucket)
+			}
+			if testCase.result.Name != result.Name {
+				t.Errorf("Test %d: Expected Object name to be %s, but instead found it to be %s", i+1, testCase.result.Name, result.Name)
+			}
+			if testCase.result.ContentType != result.ContentType {
+				t.Errorf("Test %d: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, testCase.result.ContentType, result.ContentType)
+			}
+			if testCase.result.IsDir != result.IsDir {
+				t.Errorf("Test %d: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, testCase.result.IsDir, result.IsDir)
+			}
+		}
+	}
+}
+
+// Testing getObjectInfo().
+func TestGetObjectInfoCore(t *testing.T) {
+	directory, e := ioutil.TempDir("", "minio-get-objinfo-test")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// This bucket is used for testing getObjectInfo operations.
+	err = fs.MakeBucket("test-getobjinfo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fs.PutObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resultCases := []ObjectInfo{
+		// ObjectInfo - 1.
+		// ObjectName object name set to a existing directory in the test case.
+		{Bucket: "test-getobjinfo", Name: "Asia", Size: 0, ContentType: "application/octet-stream", IsDir: true},
+		// ObjectInfo -2.
+		// ObjectName set to a existing object in the test case.
+		{Bucket: "test-getobjinfo", Name: "Asia/asiapics.jpg", Size: int64(len("asiapics")), ContentType: "image/jpeg", IsDir: false},
+		// ObjectInfo-3.
+		// Object name set to a non-existing object in the test case.
+		{Bucket: "test-getobjinfo", Name: "Africa", Size: 0, ContentType: "image/jpeg", IsDir: false},
+	}
+	testCases := []struct {
+		bucketName string
+		objectName string
+
+		// Expected output of getObjectInfo.
+		result ObjectInfo
+		err    error
+
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Testcase with object name set to a existing directory ( Test number 1).
+		{"test-getobjinfo", "Asia", resultCases[0], nil, true},
+		// ObjectName set to a existing object ( Test number 2).
+		{"test-getobjinfo", "Asia/asiapics.jpg", resultCases[1], nil, true},
+		// Object name set to a non-existing object. (Test number 3).
+		{"test-getobjinfo", "Africa", resultCases[2], fmt.Errorf("%s", filepath.FromSlash("test-getobjinfo/Africa")), false},
+	}
+	rootPath := fs.(*Filesystem).GetRootPath()
+	for i, testCase := range testCases {
+		result, err := getObjectInfo(rootPath, testCase.bucketName, testCase.objectName)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if !strings.Contains(err.Cause.Error(), testCase.err.Error()) {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error())
+			}
+		}
+
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.result.Bucket != result.Bucket {
+				t.Fatalf("Test %d: Expected Bucket name to be '%s', but found '%s' instead", i+1, testCase.result.Bucket, result.Bucket)
+			}
+			if testCase.result.Name != result.Name {
+				t.Errorf("Test %d: Expected Object name to be %s, but instead found it to be %s", i+1, testCase.result.Name, result.Name)
+			}
+			if testCase.result.ContentType != result.ContentType {
+				t.Errorf("Test %d: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, testCase.result.ContentType, result.ContentType)
+			}
+			if testCase.result.IsDir != result.IsDir {
+				t.Errorf("Test %d: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, testCase.result.IsDir, result.IsDir)
+			}
+		}
+	}
+}
+
+func BenchmarkGetObject(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark-getobject")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Make a bucket and put in a few objects.
+	err = fs.MakeBucket("bucket")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	text := "Jack and Jill went up the hill / To fetch a pail of water."
+	hasher := md5.New()
+	hasher.Write([]byte(text))
+	metadata := make(map[string]string)
+	for i := 0; i < 10; i++ {
+		metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+		_, err = fs.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		var buffer = new(bytes.Buffer)
+		r, err := fs.GetObject("bucket", "object"+strconv.Itoa(i%10), 0)
+		if err != nil {
+			b.Error(err)
+		}
+		if _, e := io.Copy(buffer, r); e != nil {
+			b.Error(e)
+		}
+		if buffer.Len() != len(text) {
+			b.Errorf("GetObject returned incorrect length %d (should be %d)\n", buffer.Len(), len(text))
+		}
+		r.Close()
+	}
+}

fs-utils.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package fs
+package main
 
 import (
 	"regexp"
@@ -51,3 +51,15 @@ func IsValidObjectName(object string) bool {
 	}
 	return true
 }
+
+// IsValidObjectPrefix verifies whether the prefix is a valid object name.
+// Its valid to have a empty prefix.
+func IsValidObjectPrefix(object string) bool {
+	// Prefix can be empty.
+	if object == "" {
+		return true
+	}
+	// Verify if prefix is a valid object name.
+	return IsValidObjectName(object)
+
+}

fs-utils_test.go

@@ -0,0 +1,114 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"testing"
+)
+
+//Validating bucket name.
+func ensureBucketName(t *testing.T, name string, testNum int, pass bool) {
+	isValidBucketName := IsValidBucketName(name)
+	if pass && !isValidBucketName {
+		t.Errorf("Test case %d: Expected \"%s\" to be a valid bucket name", testNum, name)
+	}
+	if !pass && isValidBucketName {
+		t.Errorf("Test case %d: Expected bucket name \"%s\" to be invalid", testNum, name)
+	}
+}
+
+func TestIsValidBucketName(t *testing.T) {
+	testCases := []struct {
+		bucketName string
+		shouldPass bool
+	}{
+		//cases which should pass the test
+		//passing in valid bucket names
+		{"lol", true},
+		{"1-this-is-valid", true},
+		{"1-this-too-is-valid-1", true},
+		{"this.works.too.1", true},
+		{"1234567", true},
+		{"123", true},
+		{"s3-eu-west-1.amazonaws.com", true},
+		{"ideas-are-more-powerful-than-guns", true},
+		{"testbucket", true},
+		{"1bucket", true},
+		{"bucket1", true},
+		//cases for which test should fail
+		//passing invalid bucket names
+		{"------", false},
+		{"$this-is-not-valid-too", false},
+		{"contains-$-dollar", false},
+		{"contains-^-carrot", false},
+		{"contains-$-dollar", false},
+		{"contains-$-dollar", false},
+		{"......", false},
+		{"", false},
+		{"a", false},
+		{"ab", false},
+		{".starts-with-a-dot", false},
+		{"ends-with-a-dot.", false},
+		{"ends-with-a-dash-", false},
+		{"-starts-with-a-dash", false},
+		{"THIS-BEINGS-WITH-UPPERCASe", false},
+		{"tHIS-ENDS-WITH-UPPERCASE", false},
+		{"ThisBeginsAndEndsWithUpperCase", false},
+		{"una ñina", false},
+		{"lalalallalallalalalallalallalala-theString-size-is-greater-than-64", false},
+	}
+
+	for i, testCase := range testCases {
+		ensureBucketName(t, testCase.bucketName, i+1, testCase.shouldPass)
+	}
+}
+
+//Test for validating object name.
+func ensureObjectName(t *testing.T, name string, testNum int, pass bool) {
+	isValidObjectName := IsValidObjectName(name)
+	if pass && !isValidObjectName {
+		t.Errorf("Test case %d: Expected \"%s\" to be a valid object name", testNum, name)
+	}
+	if !pass && isValidObjectName {
+		t.Errorf("Test case %d: Expected object name \"%s\" to be invalid", testNum, name)
+	}
+
+}
+
+func TestIsValidObjectName(t *testing.T) {
+	testCases := []struct {
+		objectName string
+		shouldPass bool
+	}{
+		//cases which should pass the test
+		//passing in valid object name
+		{"object", true},
+		{"The Shining Script <v1>.pdf", true},
+		{"Cost Benefit Analysis (2009-2010).pptx", true},
+		{"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", true},
+		{"SHØRT", true},
+		{"There are far too many object names, and far too few bucket names!", true},
+		//cases for which test should fail
+		//passing invalid object names
+		{"", false},
+		{string([]byte{0xff, 0xfe, 0xfd}), false},
+	}
+
+	for i, testCase := range testCases {
+		ensureObjectName(t, testCase.objectName, i+1, testCase.shouldPass)
+	}
+}

fs.go

@@ -0,0 +1,112 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// listObjectParams - list object params used for list object map
+type listObjectParams struct {
+	bucket    string
+	delimiter string
+	marker    string
+	prefix    string
+}
+
+// listMultipartObjectParams - list multipart object params used for list multipart object map
+type listMultipartObjectParams struct {
+	bucket         string
+	delimiter      string
+	keyMarker      string
+	prefix         string
+	uploadIDMarker string
+}
+
+// Filesystem - local variables
+type Filesystem struct {
+	diskPath                    string
+	minFreeDisk                 int64
+	rwLock                      *sync.RWMutex
+	listObjectMap               map[listObjectParams][]*treeWalker
+	listObjectMapMutex          *sync.Mutex
+	listMultipartObjectMap      map[listMultipartObjectParams][]<-chan multipartObjectInfo
+	listMultipartObjectMapMutex *sync.Mutex
+}
+
+// newFS instantiate a new filesystem.
+func newFS(diskPath string) (ObjectAPI, *probe.Error) {
+	fs := &Filesystem{
+		rwLock: &sync.RWMutex{},
+	}
+	fs.diskPath = diskPath
+
+	/// Defaults
+	// Minium free disk required for i/o operations to succeed.
+	fs.minFreeDisk = 5
+
+	// Initialize list object map.
+	fs.listObjectMap = make(map[listObjectParams][]*treeWalker)
+	fs.listObjectMapMutex = &sync.Mutex{}
+
+	// Initialize list multipart map.
+	fs.listMultipartObjectMap = make(map[listMultipartObjectParams][]<-chan multipartObjectInfo)
+	fs.listMultipartObjectMapMutex = &sync.Mutex{}
+
+	// Return here.
+	return fs, nil
+}
+
+func (fs Filesystem) checkBucketArg(bucket string) (string, error) {
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	bucket = getActualBucketname(fs.diskPath, bucket)
+	if status, e := isDirExist(filepath.Join(fs.diskPath, bucket)); !status {
+		if e == nil {
+			return "", BucketNotFound{Bucket: bucket}
+		} else if os.IsNotExist(e) {
+			return "", BucketNotFound{Bucket: bucket}
+		} else {
+			return "", e
+		}
+	}
+	return bucket, nil
+}
+
+func checkDiskFree(diskPath string, minFreeDisk int64) error {
+	di, e := disk.GetInfo(diskPath)
+	if e != nil {
+		return e
+	}
+	// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
+	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
+	if int64(availableDiskSpace) <= minFreeDisk {
+		return RootPathFull{Path: diskPath}
+	}
+	return nil
+}
+
+// GetRootPath - get root path.
+func (fs Filesystem) GetRootPath() string {
+	return fs.diskPath
+}

fs_api_suite_test.go

@@ -1,7 +1,5 @@
-// +build linux darwin freebsd openbsd netbsd dragonfly
-
 /*
- * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,23 +14,21 @@
  * limitations under the License.
  */
 
-package fs
+package main
 
 import (
 	"bytes"
 	"crypto/md5"
-	"encoding/base64"
 	"encoding/hex"
-	"encoding/xml"
+	"io"
 	"math/rand"
 	"strconv"
-	"time"
 
 	"gopkg.in/check.v1"
 )
 
 // APITestSuite - collection of API tests
-func APITestSuite(c *check.C, create func() Filesystem) {
+func APITestSuite(c *check.C, create func() ObjectAPI) {
 	testMakeBucket(c, create)
 	testMultipleObjectCreation(c, create)
 	testPaging(c, create)
@@ -50,21 +46,20 @@ func APITestSuite(c *check.C, create func() Filesystem) {
 	testMultipartObjectAbort(c, create)
 }
 
-func testMakeBucket(c *check.C, create func() Filesystem) {
+func testMakeBucket(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 }
 
-func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
+func testMultipartObjectCreation(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 	uploadID, err := fs.NewMultipartUpload("bucket", "key")
 	c.Assert(err, check.IsNil)
 
-	completedParts := CompleteMultipartUpload{}
-	completedParts.Part = make([]CompletePart, 0)
+	completedParts := completeMultipartUpload{}
 	for i := 1; i <= 10; i++ {
 		randomPerm := rand.Perm(10)
 		randomString := ""
@@ -74,24 +69,20 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
 
 		hasher := md5.New()
 		hasher.Write([]byte(randomString))
-		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+		expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
 
-		var calculatedmd5sum string
-		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
-			bytes.NewBufferString(randomString), nil)
+		var calculatedMD5sum string
+		calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
 		c.Assert(err, check.IsNil)
-		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
-		completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedmd5sum})
+		c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
+		completedParts.Parts = append(completedParts.Parts, completePart{PartNumber: i, ETag: calculatedMD5sum})
 	}
-	completedPartsBytes, e := xml.Marshal(completedParts)
-	c.Assert(e, check.IsNil)
-	objectMetadata, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil)
+	objInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts)
 	c.Assert(err, check.IsNil)
-	c.Assert(objectMetadata.MD5, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10")
+	c.Assert(objInfo.MD5Sum, check.Equals, "3605d84b1c43b1a664aa7c0d5082d271-10")
 }
 
-func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
+func testMultipartObjectAbort(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
@@ -99,6 +90,7 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
 	c.Assert(err, check.IsNil)
 
 	parts := make(map[int]string)
+	metadata := make(map[string]string)
 	for i := 1; i <= 10; i++ {
 		randomPerm := rand.Perm(10)
 		randomString := ""
@@ -108,21 +100,20 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
 
 		hasher := md5.New()
 		hasher.Write([]byte(randomString))
-		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+		expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
 
-		var calculatedmd5sum string
-		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
-			bytes.NewBufferString(randomString), nil)
+		metadata["md5"] = expectedMD5Sumhex
+		var calculatedMD5sum string
+		calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
 		c.Assert(err, check.IsNil)
-		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
-		parts[i] = calculatedmd5sum
+		c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
+		parts[i] = expectedMD5Sumhex
 	}
 	err = fs.AbortMultipartUpload("bucket", "key", uploadID)
 	c.Assert(err, check.IsNil)
 }
 
-func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
+func testMultipleObjectCreation(c *check.C, create func() ObjectAPI) {
 	objects := make(map[string][]byte)
 	fs := create()
 	err := fs.MakeBucket("bucket")
@@ -136,29 +127,34 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
 
 		hasher := md5.New()
 		hasher.Write([]byte(randomString))
-		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+		expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
 
 		key := "obj" + strconv.Itoa(i)
 		objects[key] = []byte(randomString)
-		objectMetadata, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil)
+		metadata := make(map[string]string)
+		metadata["md5Sum"] = expectedMD5Sumhex
+		objInfo, err := fs.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata)
 		c.Assert(err, check.IsNil)
-		c.Assert(objectMetadata.MD5, check.Equals, expectedmd5Sumhex)
+		c.Assert(objInfo.MD5Sum, check.Equals, expectedMD5Sumhex)
 	}
 
 	for key, value := range objects {
 		var byteBuffer bytes.Buffer
-		_, err := fs.GetObject(&byteBuffer, "bucket", key, 0, 0)
+		r, err := fs.GetObject("bucket", key, 0)
 		c.Assert(err, check.IsNil)
+		_, e := io.Copy(&byteBuffer, r)
+		c.Assert(e, check.IsNil)
 		c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
+		c.Assert(r.Close(), check.IsNil)
 
-		metadata, err := fs.GetObjectMetadata("bucket", key)
+		objInfo, err := fs.GetObjectInfo("bucket", key)
 		c.Assert(err, check.IsNil)
-		c.Assert(metadata.Size, check.Equals, int64(len(value)))
+		c.Assert(objInfo.Size, check.Equals, int64(len(value)))
+		r.Close()
 	}
 }
 
-func testPaging(c *check.C, create func() Filesystem) {
+func testPaging(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	fs.MakeBucket("bucket")
 	result, err := fs.ListObjects("bucket", "", "", "", 0)
@@ -168,7 +164,7 @@ func testPaging(c *check.C, create func() Filesystem) {
 	// check before paging occurs
 	for i := 0; i < 5; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
+		_, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
 		c.Assert(err, check.IsNil)
 		result, err = fs.ListObjects("bucket", "", "", "", 5)
 		c.Assert(err, check.IsNil)
@@ -178,7 +174,7 @@ func testPaging(c *check.C, create func() Filesystem) {
 	// check after paging occurs pages work
 	for i := 6; i <= 10; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
+		_, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
 		c.Assert(err, check.IsNil)
 		result, err = fs.ListObjects("bucket", "obj", "", "", 5)
 		c.Assert(err, check.IsNil)
@@ -187,9 +183,9 @@ func testPaging(c *check.C, create func() Filesystem) {
 	}
 	// check paging with prefix at end returns less objects
 	{
-		_, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
+		_, err = fs.PutObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
 		c.Assert(err, check.IsNil)
-		_, err = fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
+		_, err = fs.PutObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
 		c.Assert(err, check.IsNil)
 		result, err = fs.ListObjects("bucket", "new", "", "", 5)
 		c.Assert(err, check.IsNil)
@@ -200,35 +196,34 @@ func testPaging(c *check.C, create func() Filesystem) {
 	{
 		result, err = fs.ListObjects("bucket", "", "", "", 1000)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
 	}
 
 	// check delimited results with delimiter and prefix
 	{
-		_, err = fs.CreateObject("bucket", "this/is/delimited", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
+		_, err = fs.PutObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
 		c.Assert(err, check.IsNil)
-		_, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
+		_, err = fs.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
 		c.Assert(err, check.IsNil)
 		result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
 		c.Assert(err, check.IsNil)
 		c.Assert(len(result.Objects), check.Equals, 1)
 		c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
 	}
-	time.Sleep(time.Second)
 
 	// check delimited results with delimiter without prefix
 	{
 		result, err = fs.ListObjects("bucket", "", "", "/", 1000)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
 		c.Assert(result.Prefixes[0], check.Equals, "this/")
 	}
 
@@ -236,62 +231,57 @@ func testPaging(c *check.C, create func() Filesystem) {
 	{
 		result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[1].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj1")
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[1].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj1")
 	}
 	// check ordering of results with prefix
 	{
 		result, err = fs.ListObjects("bucket", "obj", "", "", 1000)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[1].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj10")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj2")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj3")
+		c.Assert(result.Objects[0].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[1].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj10")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj2")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj3")
 	}
 	// check ordering of results with prefix and no paging
 	{
 		result, err = fs.ListObjects("bucket", "new", "", "", 5)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
 	}
 }
 
-func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
+func testObjectOverwriteWorks(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
-	hasher1 := md5.New()
-	hasher1.Write([]byte("one"))
-	md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
-	md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
-	objectMetadata, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil)
+	_, err = fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
 	c.Assert(err, check.IsNil)
-	c.Assert(md5Sum1hex, check.Equals, objectMetadata.MD5)
+	// c.Assert(md5Sum1hex, check.Equals, objInfo.MD5Sum)
 
-	hasher2 := md5.New()
-	hasher2.Write([]byte("three"))
-	md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil))
-	_, err = fs.CreateObject("bucket", "object", md5Sum2, int64(len("three")), bytes.NewBufferString("three"), nil)
+	_, err = fs.PutObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil)
 	c.Assert(err, check.IsNil)
 
 	var bytesBuffer bytes.Buffer
-	length, err := fs.GetObject(&bytesBuffer, "bucket", "object", 0, 0)
+	r, err := fs.GetObject("bucket", "object", 0)
 	c.Assert(err, check.IsNil)
-	c.Assert(length, check.Equals, int64(len("three")))
+	_, e := io.Copy(&bytesBuffer, r)
+	c.Assert(e, check.IsNil)
 	c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three")
+	c.Assert(r.Close(), check.IsNil)
 }
 
-func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
+func testNonExistantBucketOperations(c *check.C, create func() ObjectAPI) {
 	fs := create()
-	_, err := fs.CreateObject("bucket", "object", "", int64(len("one")), bytes.NewBufferString("one"), nil)
+	_, err := fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
 	c.Assert(err, check.Not(check.IsNil))
 }
 
-func testBucketRecreateFails(c *check.C, create func() Filesystem) {
+func testBucketRecreateFails(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	err := fs.MakeBucket("string")
 	c.Assert(err, check.IsNil)
@@ -299,27 +289,25 @@ func testBucketRecreateFails(c *check.C, create func() Filesystem) {
 	c.Assert(err, check.Not(check.IsNil))
 }
 
-func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
+func testPutObjectInSubdir(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
-	hasher := md5.New()
-	hasher.Write([]byte("hello world"))
-	md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-	md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
-	objectMetadata, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
+	_, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
 	c.Assert(err, check.IsNil)
-	c.Assert(objectMetadata.MD5, check.Equals, md5Sum1hex)
 
 	var bytesBuffer bytes.Buffer
-	length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0)
+	r, err := fs.GetObject("bucket", "dir1/dir2/object", 0)
 	c.Assert(err, check.IsNil)
+	n, e := io.Copy(&bytesBuffer, r)
+	c.Assert(e, check.IsNil)
 	c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
-	c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length)
+	c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, int64(n))
+	c.Assert(r.Close(), check.IsNil)
 }
 
-func testListBuckets(c *check.C, create func() Filesystem) {
+func testListBuckets(c *check.C, create func() ObjectAPI) {
 	fs := create()
 
 	// test empty list
@@ -351,7 +339,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
 	c.Assert(err, check.IsNil)
 }
 
-func testListBucketsOrder(c *check.C, create func() Filesystem) {
+func testListBucketsOrder(c *check.C, create func() ObjectAPI) {
 	// if implementation contains a map, order of map keys will vary.
 	// this ensures they return in the same order each time
 	for i := 0; i < 10; i++ {
@@ -369,7 +357,7 @@ func testListBucketsOrder(c *check.C, create func() Filesystem) {
 	}
 }
 
-func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) {
+func testListObjectsTestsForNonExistantBucket(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	result, err := fs.ListObjects("bucket", "", "", "", 1000)
 	c.Assert(err, check.Not(check.IsNil))
@@ -377,16 +365,13 @@ func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesyst
 	c.Assert(len(result.Objects), check.Equals, 0)
 }
 
-func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
+func testNonExistantObjectInBucket(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
-	var byteBuffer bytes.Buffer
-	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
-	c.Assert(length, check.Equals, int64(0))
+	_, err = fs.GetObject("bucket", "dir1", 0)
 	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
 	switch err := err.ToGoError().(type) {
 	case ObjectNotFound:
 		c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
@@ -395,66 +380,44 @@ func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
 	}
 }
 
-func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) {
+func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
-	_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
+	_, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
 	c.Assert(err, check.IsNil)
 
-	var byteBuffer bytes.Buffer
-	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
-	c.Assert(length, check.Equals, int64(0))
+	_, err = fs.GetObject("bucket", "dir1", 0)
 	switch err := err.ToGoError().(type) {
-	case ObjectNotFound:
+	case ObjectExistsAsPrefix:
 		c.Assert(err.Bucket, check.Equals, "bucket")
-		c.Assert(err.Object, check.Equals, "dir1")
+		c.Assert(err.Prefix, check.Equals, "dir1")
 	default:
 		// force a failure with a line number
-		c.Assert(err, check.Equals, "ObjectNotFound")
+		c.Assert(err.Error(), check.Equals, "Object exists on : bucket as prefix dir1")
 	}
-	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
 
-	var byteBuffer2 bytes.Buffer
-	length, err = fs.GetObject(&byteBuffer, "bucket", "dir1/", 0, 0)
-	c.Assert(length, check.Equals, int64(0))
+	_, err = fs.GetObject("bucket", "dir1/", 0)
 	switch err := err.ToGoError().(type) {
-	case ObjectNotFound:
+	case ObjectExistsAsPrefix:
 		c.Assert(err.Bucket, check.Equals, "bucket")
-		c.Assert(err.Object, check.Equals, "dir1/")
+		c.Assert(err.Prefix, check.Equals, "dir1/")
 	default:
 		// force a failure with a line number
-		c.Assert(err, check.Equals, "ObjectNotFound")
+		c.Assert(err.Error(), check.Equals, "Object exists on : bucket as prefix dir1")
 	}
-	c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
 }
 
-func testDefaultContentType(c *check.C, create func() Filesystem) {
+func testDefaultContentType(c *check.C, create func() ObjectAPI) {
 	fs := create()
 	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
-	// test empty
-	_, err = fs.CreateObject("bucket", "one", "", int64(len("one")), bytes.NewBufferString("one"), nil)
-	metadata, err := fs.GetObjectMetadata("bucket", "one")
+	// Test empty
+	_, err = fs.PutObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil)
 	c.Assert(err, check.IsNil)
-	c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
-}
-
-func testContentMD5Set(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-
-	// test md5 invalid
-	badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA"
-	calculatedmd5sum, err := fs.CreateObject("bucket", "one", badmd5Sum, int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum)
-
-	goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA="
-	calculatedmd5sum, err = fs.CreateObject("bucket", "two", goodmd5sum, int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.IsNil)
-	c.Assert(calculatedmd5sum, check.Equals, goodmd5sum)
+	objInfo, err := fs.GetObjectInfo("bucket", "one")
+	c.Assert(err, check.IsNil)
+	c.Assert(objInfo.ContentType, check.Equals, "application/octet-stream")
 }

fs_test.go

@@ -14,29 +14,22 @@
  * limitations under the License.
  */
 
-package fs
+package main
 
 import (
 	"io/ioutil"
 	"os"
-	"testing"
 
 	. "gopkg.in/check.v1"
 )
 
-func Test(t *testing.T) { TestingT(t) }
-
-type MySuite struct{}
-
-var _ = Suite(&MySuite{})
-
-func (s *MySuite) TestAPISuite(c *C) {
+func (s *MyAPISuite) TestAPISuite(c *C) {
 	var storageList []string
-	create := func() Filesystem {
+	create := func() ObjectAPI {
 		path, e := ioutil.TempDir(os.TempDir(), "minio-")
 		c.Check(e, IsNil)
 		storageList = append(storageList, path)
-		store, err := New(path, 0)
+		store, err := newFS(path)
 		c.Check(err, IsNil)
 		return store
 	}
@@ -46,7 +39,6 @@ func (s *MySuite) TestAPISuite(c *C) {
 
 func removeRoots(c *C, roots []string) {
 	for _, root := range roots {
-		err := os.RemoveAll(root)
-		c.Check(err, IsNil)
+		os.RemoveAll(root)
 	}
 }

generic-handlers.go

@@ -17,7 +17,6 @@
 package main
 
 import (
-	"errors"
 	"net/http"
 	"path"
 	"regexp"
@@ -28,11 +27,6 @@ import (
 	"github.com/rs/cors"
 )
 
-const (
-	iso8601Format = "20060102T150405Z"
-	privateBucket = "/minio"
-)
-
 // HandlerFunc - useful to chain different middleware http.Handler
 type HandlerFunc func(http.Handler) http.Handler
 
@@ -51,8 +45,13 @@ type redirectHandler struct {
 	locationPrefix string
 }
 
+// Reserved bucket.
+const (
+	reservedBucket = "/minio"
+)
+
 func setBrowserRedirectHandler(h http.Handler) http.Handler {
-	return redirectHandler{handler: h, locationPrefix: privateBucket}
+	return redirectHandler{handler: h, locationPrefix: reservedBucket}
 }
 
 func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -84,7 +83,7 @@ func setBrowserCacheControlHandler(h http.Handler) http.Handler {
 func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if r.Method == "GET" && strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
 		// For all browser requests set appropriate Cache-Control policies
-		match, e := regexp.MatchString(privateBucket+`/([^/]+\.js|favicon.ico)`, r.URL.Path)
+		match, e := regexp.MatchString(reservedBucket+`/([^/]+\.js|favicon.ico)`, r.URL.Path)
 		if e != nil {
 			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 			return
@@ -93,7 +92,7 @@ func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			// For assets set cache expiry of one year. For each release, the name
 			// of the asset name will change and hence it can not be served from cache.
 			w.Header().Set("Cache-Control", "max-age=31536000")
-		} else if strings.HasPrefix(r.URL.Path, privateBucket+"/") {
+		} else if strings.HasPrefix(r.URL.Path, reservedBucket+"/") {
 			// For non asset requests we serve index.html which will never be cached.
 			w.Header().Set("Cache-Control", "no-store")
 		}
@@ -103,68 +102,60 @@ func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 // Adds verification for incoming paths.
 type minioPrivateBucketHandler struct {
-	handler       http.Handler
-	privateBucket string
+	handler        http.Handler
+	reservedBucket string
 }
 
 func setPrivateBucketHandler(h http.Handler) http.Handler {
-	return minioPrivateBucketHandler{handler: h, privateBucket: privateBucket}
+	return minioPrivateBucketHandler{handler: h, reservedBucket: reservedBucket}
 }
 
 func (h minioPrivateBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// For all non browser requests, reject access to 'privateBucket'.
-	if !strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && path.Clean(r.URL.Path) == privateBucket {
+	// For all non browser requests, reject access to 'reservedBucket'.
+	if !strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && path.Clean(r.URL.Path) == reservedBucket {
 		writeErrorResponse(w, r, ErrAllAccessDisabled, r.URL.Path)
 		return
 	}
 	h.handler.ServeHTTP(w, r)
 }
 
-// Supported incoming date formats.
-var timeFormats = []string{
+// Supported Amz date formats.
+var amzDateFormats = []string{
 	time.RFC1123,
 	time.RFC1123Z,
 	iso8601Format,
+	// Add new AMZ date formats here.
 }
 
-// Attempts to parse date string into known date layouts. Date layouts
-// currently supported are
-//  - ``time.RFC1123``
-//  - ``time.RFC1123Z``
-//  - ``iso8601Format``
-func parseDate(date string) (parsedTime time.Time, e error) {
-	for _, layout := range timeFormats {
-		parsedTime, e = time.Parse(layout, date)
+// parseAmzDate - parses date string into supported amz date formats.
+func parseAmzDate(amzDateStr string) (amzDate time.Time, apiErr APIErrorCode) {
+	for _, dateFormat := range amzDateFormats {
+		amzDate, e := time.Parse(dateFormat, amzDateStr)
 		if e == nil {
-			return parsedTime, nil
+			return amzDate, ErrNone
 		}
 	}
-	return time.Time{}, e
+	return time.Time{}, ErrMalformedDate
 }
 
-// Parse date string from incoming header, current supports and verifies
-// follow HTTP headers.
-//
-//  - X-Amz-Date
-//  - X-Minio-Date
-//  - Date
-//
-// In following time layouts ``time.RFC1123``, ``time.RFC1123Z`` and
-// ``iso8601Format``.
-var dateHeaders = []string{
+// Supported Amz date headers.
+var amzDateHeaders = []string{
 	"x-amz-date",
 	"x-minio-date",
 	"date",
 }
 
-func parseDateHeader(req *http.Request) (time.Time, error) {
-	for _, dateHeader := range dateHeaders {
-		date := req.Header.Get(http.CanonicalHeaderKey(dateHeader))
-		if date != "" {
-			return parseDate(date)
+// parseAmzDateHeader - parses supported amz date headers, in
+// supported amz date formats.
+func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
+	for _, amzDateHeader := range amzDateHeaders {
+		amzDateStr := req.Header.Get(http.CanonicalHeaderKey(amzDateHeader))
+		if amzDateStr != "" {
+			return parseAmzDate(amzDateStr)
 		}
 	}
-	return time.Time{}, errors.New("Date header missing, invalid request.")
+	// Date header missing.
+	return time.Time{}, ErrMissingDateHeader
 }
 
 type timeHandler struct {
@@ -178,18 +169,18 @@ func setTimeValidityHandler(h http.Handler) http.Handler {
 
 func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Verify if date headers are set, if not reject the request
-	if r.Header.Get("Authorization") != "" {
-		date, e := parseDateHeader(r)
-		if e != nil {
+	if _, ok := r.Header["Authorization"]; ok {
+		amzDate, apiErr := parseAmzDateHeader(r)
+		if apiErr != ErrNone {
 			// All our internal APIs are sensitive towards Date
 			// header, for all requests where Date header is not
 			// present we will reject such clients.
-			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
+			writeErrorResponse(w, r, apiErr, r.URL.Path)
 			return
 		}
 		// Verify if the request date header is more than 5minutes
 		// late, reject such clients.
-		if time.Now().UTC().Sub(date)/time.Minute > time.Duration(5)*time.Minute {
+		if time.Now().UTC().Sub(amzDate)/time.Minute > time.Duration(5)*time.Minute {
 			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
 			return
 		}

globals.go

@@ -16,7 +16,55 @@
 
 package main
 
+import (
+	"github.com/fatih/color"
+	"github.com/minio/cli"
+	"github.com/minio/mc/pkg/console"
+)
+
 // Global constants for Minio.
 const (
 	minGoVersion = ">= 1.6" // Minio requires at least Go v1.6
 )
+
+// minio configuration related constants.
+const (
+	globalMinioConfigVersion = "4"
+	globalMinioConfigDir     = ".minio"
+	globalMinioCertsDir      = ".minio/certs"
+	globalMinioCertFile      = "public.crt"
+	globalMinioKeyFile       = "private.key"
+	globalMinioConfigFile    = "config.json"
+)
+
+var (
+	globalQuiet = false // Quiet flag set via command line
+	globalDebug = false // Debug flag set via command line
+	// Add new global flags here.
+)
+
+// global colors.
+var (
+	colorMagenta = color.New(color.FgMagenta, color.Bold).SprintfFunc()
+	colorWhite   = color.New(color.FgWhite, color.Bold).SprintfFunc()
+	colorGreen   = color.New(color.FgGreen, color.Bold).SprintfFunc()
+)
+
+// Set global states. NOTE: It is deliberately kept monolithic to
+// ensure we dont miss out any flags.
+func setGlobals(quiet, debug bool) {
+	globalQuiet = quiet
+	globalDebug = debug
+	// Enable debug messages if requested.
+	if globalDebug {
+		console.DebugPrint = true
+	}
+}
+
+// Set global states. NOTE: It is deliberately kept monolithic to
+// ensure we dont miss out any flags.
+func setGlobalsFromContext(ctx *cli.Context) {
+	quiet := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
+	debug := ctx.Bool("debug") || ctx.GlobalBool("debug")
+	setGlobals(quiet, debug)
+}

httprange.go

@@ -22,7 +22,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/minio/pkg/fs"
 	"github.com/minio/minio/pkg/probe"
 )
 
@@ -60,7 +59,7 @@ func getRequestedRange(hrange string, size int64) (*httpRange, *probe.Error) {
 func (r *httpRange) parse(ra string) *probe.Error {
 	i := strings.Index(ra, "-")
 	if i < 0 {
-		return probe.NewError(fs.InvalidRange{})
+		return probe.NewError(InvalidRange{})
 	}
 	start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:])
 	if start == "" {
@@ -68,7 +67,7 @@ func (r *httpRange) parse(ra string) *probe.Error {
 		// range start relative to the end of the file.
 		i, err := strconv.ParseInt(end, 10, 64)
 		if err != nil {
-			return probe.NewError(fs.InvalidRange{})
+			return probe.NewError(InvalidRange{})
 		}
 		if i > r.size {
 			i = r.size
@@ -78,7 +77,7 @@ func (r *httpRange) parse(ra string) *probe.Error {
 	} else {
 		i, err := strconv.ParseInt(start, 10, 64)
 		if err != nil || i > r.size || i < 0 {
-			return probe.NewError(fs.InvalidRange{})
+			return probe.NewError(InvalidRange{})
 		}
 		r.start = i
 		if end == "" {
@@ -87,7 +86,7 @@ func (r *httpRange) parse(ra string) *probe.Error {
 		} else {
 			i, err := strconv.ParseInt(end, 10, 64)
 			if err != nil || r.start > i {
-				return probe.NewError(fs.InvalidRange{})
+				return probe.NewError(InvalidRange{})
 			}
 			if i >= r.size {
 				i = r.size - 1
@@ -104,7 +103,7 @@ func (r *httpRange) parseRange(s string) *probe.Error {
 		return probe.NewError(errors.New("header not present"))
 	}
 	if !strings.HasPrefix(s, b) {
-		return probe.NewError(fs.InvalidRange{})
+		return probe.NewError(InvalidRange{})
 	}
 
 	ras := strings.Split(s[len(b):], ",")
@@ -118,7 +117,7 @@ func (r *httpRange) parseRange(s string) *probe.Error {
 
 	ra := strings.TrimSpace(ras[0])
 	if ra == "" {
-		return probe.NewError(fs.InvalidRange{})
+		return probe.NewError(InvalidRange{})
 	}
 	return r.parse(ra)
 }

logger-console-hook.go

@@ -0,0 +1,46 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// consoleLogger - default logger if not other logging is enabled.
+type consoleLogger struct {
+	Enable bool   `json:"enable"`
+	Level  string `json:"level"`
+}
+
+// enable console logger.
+func enableConsoleLogger() {
+	clogger := serverConfig.GetConsoleLogger()
+	if !clogger.Enable {
+		// Disable console logger if asked for.
+		log.Out = ioutil.Discard
+		return
+	}
+	// log.Out and log.Formatter use the default versions.
+	// Only set specific log level.
+	lvl, e := logrus.ParseLevel(clogger.Level)
+	fatalIf(probe.NewError(e), "Unknown log level detected, please fix your console logger configuration.", nil)
+
+	log.Level = lvl
+}

logger-file-hook.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,29 +24,28 @@ import (
 	"github.com/minio/minio/pkg/probe"
 )
 
+type fileLogger struct {
+	Enable   bool   `json:"enable"`
+	Filename string `json:"fileName"`
+	Level    string `json:"level"`
+}
+
 type localFile struct {
 	*os.File
 }
 
-func log2File(filename string) *probe.Error {
-	fileHook, e := newFile(filename)
-	if e != nil {
-		return probe.NewError(e)
-	}
-	log.Hooks.Add(fileHook)                 // Add a local file hook.
-	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
-	log.Level = logrus.InfoLevel            // Minimum log level.
-	return nil
-}
-
-func newFile(filename string) (*localFile, error) {
-	file, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
-	if err != nil {
-		return nil, err
-	}
-	return &localFile{file}, nil
+func enableFileLogger(filename string) {
+	file, e := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	fatalIf(probe.NewError(e), "Unable to open log file.", nil)
+
+	// Add a local file hook.
+	log.Hooks.Add(&localFile{file})
+	// Set default JSON formatter.
+	log.Formatter = new(logrus.JSONFormatter)
+	log.Level = logrus.InfoLevel // Minimum log level.
 }
 
+// Fire fires the file logger hook and logs to the file.
 func (l *localFile) Fire(entry *logrus.Entry) error {
 	line, err := entry.String()
 	if err != nil {

logger-mongo-hook.go

@@ -1,75 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"fmt"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio/pkg/probe"
-	"gopkg.in/mgo.v2"
-	"gopkg.in/mgo.v2/bson"
-)
-
-// mongoDB collection
-type mongoDB struct {
-	c *mgo.Collection
-}
-
-func log2Mongo(url, db, collection string) *probe.Error {
-	mongoHook, e := newMongo(url, db, collection)
-	if e != nil {
-		return probe.NewError(e)
-	}
-	log.Hooks.Add(mongoHook)                // Add mongodb hook.
-	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
-	log.Level = logrus.InfoLevel            // Minimum log level.
-	return nil
-}
-
-// newMongo -
-func newMongo(mgoEndpoint, db, collection string) (*mongoDB, error) {
-	session, err := mgo.Dial(mgoEndpoint)
-	if err != nil {
-		return nil, err
-	}
-	return &mongoDB{c: session.DB(db).C(collection)}, nil
-}
-
-// Fire - the log event
-func (h *mongoDB) Fire(entry *logrus.Entry) error {
-	entry.Data["Level"] = entry.Level.String()
-	entry.Data["Time"] = entry.Time
-	entry.Data["Message"] = entry.Message
-	mgoErr := h.c.Insert(bson.M(entry.Data))
-	if mgoErr != nil {
-		return fmt.Errorf("Failed to send log entry to mongodb: %s", mgoErr)
-	}
-	return nil
-}
-
-// Levels -
-func (h *mongoDB) Levels() []logrus.Level {
-	return []logrus.Level{
-		logrus.PanicLevel,
-		logrus.FatalLevel,
-		logrus.ErrorLevel,
-		logrus.WarnLevel,
-		logrus.InfoLevel,
-		logrus.DebugLevel,
-	}
-}

logger-syslog-hook_nix.go

@@ -1,7 +1,7 @@
 // +build !windows
 
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,6 +26,12 @@ import (
 	"github.com/minio/minio/pkg/probe"
 )
 
+type syslogLogger struct {
+	Enable bool   `json:"enable"`
+	Addr   string `json:"address"`
+	Level  string `json:"level"`
+}
+
 // syslogHook to send logs via syslog.
 type syslogHook struct {
 	writer        *syslog.Writer
@@ -33,28 +39,27 @@ type syslogHook struct {
 	syslogRaddr   string
 }
 
-func log2Syslog(network, raddr string) *probe.Error {
-	syslogHook, e := newSyslog(network, raddr, syslog.LOG_ERR, "MINIO")
-	if e != nil {
-		return probe.NewError(e)
-	}
+// enableSyslogLogger - enable logger at raddr.
+func enableSyslogLogger(raddr string) {
+	syslogHook, e := newSyslog("udp", raddr, syslog.LOG_ERR, "MINIO")
+	fatalIf(probe.NewError(e), "Unable to instantiate syslog.", nil)
+
 	log.Hooks.Add(syslogHook)               // Add syslog hook.
 	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
 	log.Level = logrus.InfoLevel            // Minimum log level.
-	return nil
 }
 
 // newSyslog - Creates a hook to be added to an instance of logger.
 func newSyslog(network, raddr string, priority syslog.Priority, tag string) (*syslogHook, error) {
-	w, err := syslog.Dial(network, raddr, priority, tag)
-	return &syslogHook{w, network, raddr}, err
+	w, e := syslog.Dial(network, raddr, priority, tag)
+	return &syslogHook{w, network, raddr}, e
 }
 
 // Fire - fire the log event
 func (hook *syslogHook) Fire(entry *logrus.Entry) error {
-	line, err := entry.String()
-	if err != nil {
-		return fmt.Errorf("Unable to read entry, %v", err)
+	line, e := entry.String()
+	if e != nil {
+		return fmt.Errorf("Unable to read entry, %v", e)
 	}
 	switch entry.Level {
 	case logrus.PanicLevel:

logger-syslog-hook_windows.go

@@ -1,7 +1,7 @@
 // +build windows
 
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,13 @@ package main
 
 import "github.com/minio/minio/pkg/probe"
 
-func log2Syslog(network, raddr string) *probe.Error {
-	return probe.NewError(errSysLogNotSupported)
+type syslogLogger struct {
+	Enable bool   `json:"enable"`
+	Addr   string `json:"address"`
+	Level  string `json:"level"`
+}
+
+// enableSyslogLogger - unsupported on windows.
+func enableSyslogLogger(raddr string) {
+	fatalIf(probe.NewError(errSyslogNotSupported), "Unable to enable syslog.", nil)
 }

logger.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,6 +28,21 @@ type fields map[string]interface{}
 
 var log = logrus.New() // Default console logger.
 
+// logger carries logging configuration for various supported loggers.
+// Currently supported loggers are
+//
+//   - console [default]
+//   - file
+//   - syslog
+//
+type logger struct {
+	Console consoleLogger `json:"console"`
+	File    fileLogger    `json:"file"`
+	Syslog  syslogLogger  `json:"syslog"`
+	// Add new loggers here.
+}
+
+// errorIf synonymous with fatalIf but doesn't exit on error != nil
 func errorIf(err *probe.Error, msg string, fields map[string]interface{}) {
 	if err == nil {
 		return
@@ -49,6 +64,7 @@ func errorIf(err *probe.Error, msg string, fields map[string]interface{}) {
 	log.WithFields(fields).Error(msg)
 }
 
+// fatalIf wrapper function which takes error and prints jsonic error messages.
 func fatalIf(err *probe.Error, msg string, fields map[string]interface{}) {
 	if err == nil {
 		return

main.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,17 @@ import (
 	"github.com/minio/cli"
 	"github.com/minio/mc/pkg/console"
 	"github.com/minio/minio/pkg/probe"
+	"github.com/olekukonko/ts"
+)
+
+var (
+	// global flags for minio.
+	minioFlags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "help, h",
+			Usage: "Show help.",
+		},
+	}
 )
 
 // Help template for minio.
@@ -62,12 +73,20 @@ func init() {
 	if !isContainerized() && os.Geteuid() == 0 {
 		console.Fatalln("Please run ‘minio’ as a non-root user.")
 	}
-
 }
 
 func migrate() {
 	// Migrate config file
 	migrateConfig()
+
+	// Migrate other configs here.
+}
+
+func enableLoggers() {
+	// Enable all loggers here.
+	enableConsoleLogger()
+
+	// Add your logger here.
 }
 
 // Tries to get os/arch/platform specific information
@@ -118,26 +137,18 @@ func findClosestCommands(command string) []string {
 }
 
 func registerApp() *cli.App {
-	// register all commands
+	// Register all commands.
 	registerCommand(serverCmd)
-	registerCommand(configCmd)
 	registerCommand(versionCmd)
 	registerCommand(updateCmd)
 
-	// register all flags
-	registerFlag(configFolderFlag)
-	registerFlag(addressFlag)
-	registerFlag(accessLogFlag)
-	registerFlag(certFlag)
-	registerFlag(keyFlag)
-
-	// set up app
+	// Set up app.
 	app := cli.NewApp()
 	app.Name = "Minio"
 	app.Author = "Minio.io"
-	app.Usage = "Cloud Storage Server for Micro Services."
+	app.Usage = "Distributed Object Storage Server for Micro Services."
 	app.Description = `Micro services environment provisions one Minio server per application instance. Scalability is achieved through large number of smaller personalized instances. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.`
-	app.Flags = flags
+	app.Flags = append(minioFlags, globalFlags...)
 	app.Commands = commands
 	app.CustomAppHelpTemplate = minioHelpTemplate
 	app.CommandNotFound = func(ctx *cli.Context, command string) {
@@ -160,7 +171,7 @@ func checkMainSyntax(c *cli.Context) {
 		console.Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
 	}
 	if configPath == "" {
-		console.Fatalln("Config folder cannot be empty, please specify --config-folder <foldername>.")
+		console.Fatalln("Config folder cannot be empty, please specify --config-dir <foldername>.")
 	}
 }
 
@@ -171,8 +182,11 @@ func main() {
 
 	app := registerApp()
 	app.Before = func(c *cli.Context) error {
+		// Set global flags.
+		setGlobalsFromContext(c)
+
 		// Sets new config folder.
-		setGlobalConfigPath(c.GlobalString("config-folder"))
+		setGlobalConfigPath(c.GlobalString("config-dir"))
 
 		// Valid input arguments to main.
 		checkMainSyntax(c)
@@ -180,11 +194,36 @@ func main() {
 		// Migrate any old version of config / state files to newer format.
 		migrate()
 
+		// Initialize config.
+		err := initConfig()
+		fatalIf(err.Trace(), "Unable to initialize minio config.", nil)
+
+		// Enable all loggers by now.
+		enableLoggers()
+
+		// Do not print update messages, if quiet flag is set.
+		if !globalQuiet {
+			// Do not print any errors in release update function.
+			noError := true
+			updateMsg := getReleaseUpdate(minioUpdateStableURL, noError)
+			if updateMsg.Update {
+				console.Println(updateMsg)
+			}
+		}
+
+		// Return here.
 		return nil
 	}
 	app.ExtraInfo = func() map[string]string {
-		return getSystemData()
+		if _, e := ts.GetSize(); e != nil {
+			globalQuiet = true
+		}
+		// Enable if debug is enabled.
+		if globalDebug {
+			return getSystemData()
+		}
+		return make(map[string]string)
 	}
-
+	// Run the app - exit on error.
 	app.RunAndExitOnError()
 }

minio.md

@@ -0,0 +1,40 @@
+# Server command line SPEC - Fri Apr  1 19:49:56 PDT 2016
+
+## Single disk
+Regular single server, single disk mode.
+```
+$ minio server <disk>
+```
+
+##  Multi disk
+```
+$ minio controller start - Start minio controller.
+```
+Prints a secret token to be used by all parties.
+
+Start all servers without disk with `MINIO_CONTROLLER` env set, start in cluster mode.
+```
+$ MINIO_CONTROLLER=<host>:<token> minio server
+```
+
+## Minio Controller cli.
+
+Set controller host and token.
+```
+$ export MINIO_CONTROLLER=<host>:<token>
+```
+
+Create a cluster from the pool of nodes.
+```
+$ minioctl new <clustername> <ip1>:/mnt/disk1 .. <ip16>:/mnt/disk1
+```
+
+Start the cluster.
+```
+$ minioctl start <clustername>
+```
+
+Stop the cluster
+```
+$ minioctl stop <clustername>
+```

notifier.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"fmt"
-	"math"
 	"runtime"
 	"strings"
 
@@ -29,25 +28,22 @@ import (
 
 // colorizeUpdateMessage - inspired from Yeoman project npm package https://github.com/yeoman/update-notifier
 func colorizeUpdateMessage(updateString string) (string, *probe.Error) {
-	// initialize coloring
+	// Initialize coloring.
 	cyan := color.New(color.FgCyan, color.Bold).SprintFunc()
 	yellow := color.New(color.FgYellow, color.Bold).SprintfFunc()
 
-	// calculate length without color coding, due to ANSI color characters padded to actual
-	// string the final length is wrong than the original string length
-	line1Str := fmt.Sprintf("  Update available: ")
-	line2Str := fmt.Sprintf("  Run \"%s\" to update. ", updateString)
+	// Calculate length without color coding, due to ANSI color
+	// characters padded to actual string the final length is wrong
+	// than the original string length.
+	line1Str := fmt.Sprintf("  New update: %s ", updateString)
 	line1Length := len(line1Str)
-	line2Length := len(line2Str)
 
-	// populate lines with color coding
-	line1InColor := line1Str
-	line2InColor := fmt.Sprintf("  Run \"%s\" to update. ", cyan(updateString))
+	// Populate lines with color coding.
+	line1InColor := fmt.Sprintf("  New update: %s ", cyan(updateString))
 
-	// calculate the rectangular box size
-	maxContentWidth := int(math.Max(float64(line1Length), float64(line2Length)))
+	// Calculate the rectangular box size.
+	maxContentWidth := line1Length
 	line1Rest := maxContentWidth - line1Length
-	line2Rest := maxContentWidth - line2Length
 
 	terminal, err := ts.GetSize()
 	if err != nil {
@@ -56,31 +52,29 @@ func colorizeUpdateMessage(updateString string) (string, *probe.Error) {
 
 	var message string
 	switch {
-	case len(line2Str) > terminal.Col():
-		message = "\n" + line1InColor + "\n" + line2InColor + "\n"
+	case len(line1Str) > terminal.Col():
+		message = "\n" + line1InColor + "\n"
 	default:
-		// on windows terminal turn off unicode characters
+		// On windows terminal turn off unicode characters.
 		var top, bottom, sideBar string
 		if runtime.GOOS == "windows" {
 			top = yellow("*" + strings.Repeat("*", maxContentWidth) + "*")
 			bottom = yellow("*" + strings.Repeat("*", maxContentWidth) + "*")
 			sideBar = yellow("|")
 		} else {
-			// color the rectangular box, use unicode characters here
+			// Color the rectangular box, use unicode characters here.
 			top = yellow("┏" + strings.Repeat("━", maxContentWidth) + "┓")
 			bottom = yellow("┗" + strings.Repeat("━", maxContentWidth) + "┛")
 			sideBar = yellow("┃")
 		}
-		// fill spaces to the rest of the area
+		// Fill spaces to the rest of the area.
 		spacePaddingLine1 := strings.Repeat(" ", line1Rest)
-		spacePaddingLine2 := strings.Repeat(" ", line2Rest)
 
-		// construct the final message
+		// Construct the final message.
 		message = "\n" + top + "\n" +
 			sideBar + line1InColor + spacePaddingLine1 + sideBar + "\n" +
-			sideBar + line2InColor + spacePaddingLine2 + sideBar + "\n" +
 			bottom + "\n"
 	}
-	// return the final message
+	// Return the final message.
 	return message, nil
 }

object-api-interface.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"io"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+// ObjectAPI interface.
+type ObjectAPI interface {
+	// Bucket resource API.
+	DeleteBucket(bucket string) *probe.Error
+	ListBuckets() ([]BucketInfo, *probe.Error)
+	MakeBucket(bucket string) *probe.Error
+	GetBucketInfo(bucket string) (BucketInfo, *probe.Error)
+
+	// Bucket query API.
+	ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error)
+	ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error)
+
+	// Object resource API.
+	GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error)
+	GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error)
+	PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error)
+	DeleteObject(bucket, object string) *probe.Error
+
+	// Object query API.
+	NewMultipartUpload(bucket, object string) (string, *probe.Error)
+	PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error)
+	ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error)
+	CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (ObjectInfo, *probe.Error)
+	AbortMultipartUpload(bucket, object, uploadID string) *probe.Error
+}

object-datatypes.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,41 +14,30 @@
  * limitations under the License.
  */
 
-package fs
+package main
 
-import (
-	"os"
-	"time"
-)
+import "time"
 
-// BucketMetadata - name and create date
-type BucketMetadata struct {
+// BucketInfo - bucket name and create date
+type BucketInfo struct {
 	Name    string
 	Created time.Time
 }
 
-// ObjectMetadata - object key and its relevant metadata
-type ObjectMetadata struct {
-	Bucket string
-	Object string
-
+// ObjectInfo - object info.
+type ObjectInfo struct {
+	Bucket       string
+	Name         string
+	ModifiedTime time.Time
 	ContentType  string
-	LastModified time.Time
-	Mode         os.FileMode
-	MD5          string
+	MD5Sum       string
 	Size         int64
+	IsDir        bool
+	Err          error
 }
 
-// PartMetadata - various types of individual part resources
-type PartMetadata struct {
-	PartNumber   int
-	LastModified time.Time
-	ETag         string
-	Size         int64
-}
-
-// ObjectResourcesMetadata - various types of object resources
-type ObjectResourcesMetadata struct {
+// ListPartsInfo - various types of object resources.
+type ListPartsInfo struct {
 	Bucket               string
 	Object               string
 	UploadID             string
@@ -58,20 +47,12 @@ type ObjectResourcesMetadata struct {
 	MaxParts             int
 	IsTruncated          bool
 
-	Part         []PartMetadata
+	Parts        []partInfo
 	EncodingType string
 }
 
-// UploadMetadata container capturing metadata on in progress multipart upload in a given bucket
-type UploadMetadata struct {
-	Object       string
-	UploadID     string
-	StorageClass string
-	Initiated    time.Time
-}
-
-// BucketMultipartResourcesMetadata - various types of bucket resources for inprogress multipart uploads
-type BucketMultipartResourcesMetadata struct {
+// ListMultipartsInfo - various types of bucket resources for inprogress multipart uploads.
+type ListMultipartsInfo struct {
 	KeyMarker          string
 	UploadIDMarker     string
 	NextKeyMarker      string
@@ -79,34 +60,50 @@ type BucketMultipartResourcesMetadata struct {
 	EncodingType       string
 	MaxUploads         int
 	IsTruncated        bool
-	Upload             []*UploadMetadata
+	Uploads            []uploadMetadata
 	Prefix             string
 	Delimiter          string
 	CommonPrefixes     []string
 }
 
-// ListObjectsResult - container for list object request results.
-type ListObjectsResult struct {
+// ListObjectsInfo - container for list objects.
+type ListObjectsInfo struct {
 	IsTruncated bool
 	NextMarker  string
-	Objects     []ObjectMetadata
+	Objects     []ObjectInfo
 	Prefixes    []string
 }
 
-// CompletePart - completed part container
-type CompletePart struct {
+// partInfo - various types of individual part resources.
+type partInfo struct {
+	PartNumber   int
+	LastModified time.Time
+	ETag         string
+	Size         int64
+}
+
+// uploadMetadata container capturing metadata on in progress multipart upload in a given bucket
+type uploadMetadata struct {
+	Object       string
+	UploadID     string
+	StorageClass string
+	Initiated    time.Time
+}
+
+// completePart - completed part container.
+type completePart struct {
 	PartNumber int
 	ETag       string
 }
 
 // completedParts is a sortable interface for Part slice
-type completedParts []CompletePart
+type completedParts []completePart
 
 func (a completedParts) Len() int           { return len(a) }
 func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
 
-// CompleteMultipartUpload container for completing multipart upload
-type CompleteMultipartUpload struct {
-	Part []CompletePart
+// completeMultipartUpload container for completing multipart upload
+type completeMultipartUpload struct {
+	Parts []completePart `xml:"Part"`
 }

object-interface.go

@@ -0,0 +1 @@
+package main

pkg/crypto/sha1/.gitignore

@@ -1 +0,0 @@
-*.syso

pkg/crypto/sha1/sha1.go

@@ -1,150 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file of
-// Golang project:
-//    https://github.com/golang/go/blob/master/LICENSE
-
-// Using this part of Minio codebase under the license
-// Apache License Version 2.0 with modifications
-
-// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174.
-package sha1
-
-import "hash"
-
-// Size - The size of a SHA1 checksum in bytes.
-const Size = 20
-
-// BlockSize - The blocksize of SHA1 in bytes.
-const BlockSize = 64
-
-const (
-	chunk = 64
-	init0 = 0x67452301
-	init1 = 0xEFCDAB89
-	init2 = 0x98BADCFE
-	init3 = 0x10325476
-	init4 = 0xC3D2E1F0
-)
-
-// digest represents the partial evaluation of a checksum.
-type digest struct {
-	h   [5]uint32
-	x   [chunk]byte
-	nx  int
-	len uint64
-}
-
-// Reset digest
-func (d *digest) Reset() {
-	d.h[0] = init0
-	d.h[1] = init1
-	d.h[2] = init2
-	d.h[3] = init3
-	d.h[4] = init4
-	d.nx = 0
-	d.len = 0
-}
-
-// New returns a new hash.Hash computing the SHA1 checksum.
-func New() hash.Hash {
-	d := new(digest)
-	d.Reset()
-	return d
-}
-
-// Return output size
-func (d *digest) Size() int { return Size }
-
-// Return checksum blocksize
-func (d *digest) BlockSize() int { return BlockSize }
-
-// Write to digest
-func (d *digest) Write(p []byte) (nn int, err error) {
-	nn = len(p)
-	d.len += uint64(nn)
-	if d.nx > 0 {
-		n := copy(d.x[d.nx:], p)
-		d.nx += n
-		if d.nx == chunk {
-			block(d, d.x[:])
-			d.nx = 0
-		}
-		p = p[n:]
-	}
-	if len(p) >= chunk {
-		n := len(p) &^ (chunk - 1)
-		block(d, p[:n])
-		p = p[n:]
-	}
-	if len(p) > 0 {
-		d.nx = copy(d.x[:], p)
-	}
-	return
-}
-
-// Return checksum bytes
-func (d *digest) Sum(in []byte) []byte {
-	// Make a copy of d0 so that caller can keep writing and summing.
-	d0 := *d
-	hash := d0.checkSum()
-	return append(in, hash[:]...)
-}
-
-// Intermediate checksum function
-func (d *digest) checkSum() [Size]byte {
-	len := d.len
-	// Padding.  Add a 1 bit and 0 bits until 56 bytes mod 64.
-	var tmp [64]byte
-	tmp[0] = 0x80
-	if len%64 < 56 {
-		d.Write(tmp[0 : 56-len%64])
-	} else {
-		d.Write(tmp[0 : 64+56-len%64])
-	}
-
-	// Length in bits.
-	len <<= 3
-	for i := uint(0); i < 8; i++ {
-		tmp[i] = byte(len >> (56 - 8*i))
-	}
-	d.Write(tmp[0:8])
-
-	if d.nx != 0 {
-		panic("d.nx != 0")
-	}
-
-	var digest [Size]byte
-	for i, s := range d.h {
-		digest[i*4] = byte(s >> 24)
-		digest[i*4+1] = byte(s >> 16)
-		digest[i*4+2] = byte(s >> 8)
-		digest[i*4+3] = byte(s)
-	}
-
-	return digest
-}
-
-// Sum - single caller sha1 helper
-func Sum(data []byte) [Size]byte {
-	var d digest
-	d.Reset()
-	d.Write(data)
-	return d.checkSum()
-}

pkg/crypto/sha1/sha1_linux.S

@@ -1,967 +0,0 @@
-/*
- *	Implement fast SHA-1 with AVX2 instructions. (x86_64)
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2014 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Contact Information:
- * Ilya Albrekht <ilya.albrekht@intel.com>
- * Maxim Locktyukhin <maxim.locktyukhin@intel.com>
- * Ronen Zohar <ronen.zohar@intel.com>
- * Chandramouli Narayanan <mouli@linux.intel.com>
- *
- * BSD LICENSE
- *
- * Copyright(c) 2014 Intel Corporation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/*
- * SHA-1 implementation with Intel(R) AVX2 instruction set extensions.
- *
- *This implementation is based on the previous SSSE3 release:
- * https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
- *
- *Updates 20-byte SHA-1 record in 'hash' for even number of
- *'num_blocks' consecutive 64-byte blocks
- *
- */
-
-/*
- * Using this part of Minio codebase under the license
- * Apache License Version 2.0 with modifications
- *
- */
-
-#ifdef HAS_AVX2
-#ifndef ENTRY
-#define ENTRY(name) \
-        .globl name             ; \
-        .align 4,0x90           ; \
-        name:
-#endif
-
-#ifndef END
-#define END(name) \
-        .size name, .-name
-#endif
-
-#ifndef ENDPROC
-#define ENDPROC(name) \
-        .type name, @function   ; \
-        END(name)
-#endif
-
-#define NUM_INVALID		100
-
-#define TYPE_R32		0
-#define TYPE_R64		1
-#define TYPE_XMM		2
-#define TYPE_INVALID	100
-
-	.macro R32_NUM opd r32
-	\opd = NUM_INVALID
-	.ifc \r32,%eax
-	\opd = 0
-	.endif
-	.ifc \r32,%ecx
-	\opd = 1
-	.endif
-	.ifc \r32,%edx
-	\opd = 2
-	.endif
-	.ifc \r32,%ebx
-	\opd = 3
-	.endif
-	.ifc \r32,%esp
-	\opd = 4
-	.endif
-	.ifc \r32,%ebp
-	\opd = 5
-	.endif
-	.ifc \r32,%esi
-	\opd = 6
-	.endif
-	.ifc \r32,%edi
-	\opd = 7
-	.endif
-#ifdef X86_64
-	.ifc \r32,%r8d
-	\opd = 8
-	.endif
-	.ifc \r32,%r9d
-	\opd = 9
-	.endif
-	.ifc \r32,%r10d
-	\opd = 10
-	.endif
-	.ifc \r32,%r11d
-	\opd = 11
-	.endif
-	.ifc \r32,%r12d
-	\opd = 12
-	.endif
-	.ifc \r32,%r13d
-	\opd = 13
-	.endif
-	.ifc \r32,%r14d
-	\opd = 14
-	.endif
-	.ifc \r32,%r15d
-	\opd = 15
-	.endif
-#endif
-	.endm
-
-	.macro R64_NUM opd r64
-	\opd = NUM_INVALID
-#ifdef X86_64
-	.ifc \r64,%rax
-	\opd = 0
-	.endif
-	.ifc \r64,%rcx
-	\opd = 1
-	.endif
-	.ifc \r64,%rdx
-	\opd = 2
-	.endif
-	.ifc \r64,%rbx
-	\opd = 3
-	.endif
-	.ifc \r64,%rsp
-	\opd = 4
-	.endif
-	.ifc \r64,%rbp
-	\opd = 5
-	.endif
-	.ifc \r64,%rsi
-	\opd = 6
-	.endif
-	.ifc \r64,%rdi
-	\opd = 7
-	.endif
-	.ifc \r64,%r8
-	\opd = 8
-	.endif
-	.ifc \r64,%r9
-	\opd = 9
-	.endif
-	.ifc \r64,%r10
-	\opd = 10
-	.endif
-	.ifc \r64,%r11
-	\opd = 11
-	.endif
-	.ifc \r64,%r12
-	\opd = 12
-	.endif
-	.ifc \r64,%r13
-	\opd = 13
-	.endif
-	.ifc \r64,%r14
-	\opd = 14
-	.endif
-	.ifc \r64,%r15
-	\opd = 15
-	.endif
-#endif
-	.endm
-
-	.macro XMM_NUM opd xmm
-	\opd = NUM_INVALID
-	.ifc \xmm,%xmm0
-	\opd = 0
-	.endif
-	.ifc \xmm,%xmm1
-	\opd = 1
-	.endif
-	.ifc \xmm,%xmm2
-	\opd = 2
-	.endif
-	.ifc \xmm,%xmm3
-	\opd = 3
-	.endif
-	.ifc \xmm,%xmm4
-	\opd = 4
-	.endif
-	.ifc \xmm,%xmm5
-	\opd = 5
-	.endif
-	.ifc \xmm,%xmm6
-	\opd = 6
-	.endif
-	.ifc \xmm,%xmm7
-	\opd = 7
-	.endif
-	.ifc \xmm,%xmm8
-	\opd = 8
-	.endif
-	.ifc \xmm,%xmm9
-	\opd = 9
-	.endif
-	.ifc \xmm,%xmm10
-	\opd = 10
-	.endif
-	.ifc \xmm,%xmm11
-	\opd = 11
-	.endif
-	.ifc \xmm,%xmm12
-	\opd = 12
-	.endif
-	.ifc \xmm,%xmm13
-	\opd = 13
-	.endif
-	.ifc \xmm,%xmm14
-	\opd = 14
-	.endif
-	.ifc \xmm,%xmm15
-	\opd = 15
-	.endif
-	.endm
-
-	.macro TYPE type reg
-	R32_NUM reg_type_r32 \reg
-	R64_NUM reg_type_r64 \reg
-	XMM_NUM reg_type_xmm \reg
-	.if reg_type_r64 <> NUM_INVALID
-	\type = TYPE_R64
-	.elseif reg_type_r32 <> NUM_INVALID
-	\type = TYPE_R32
-	.elseif reg_type_xmm <> NUM_INVALID
-	\type = TYPE_XMM
-	.else
-	\type = TYPE_INVALID
-	.endif
-	.endm
-
-	.macro PFX_OPD_SIZE
-	.byte 0x66
-	.endm
-
-	.macro PFX_REX opd1 opd2 W=0
-	.if ((\opd1 | \opd2) & 8) || \W
-	.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
-	.endif
-	.endm
-
-	.macro MODRM mod opd1 opd2
-	.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
-	.endm
-
-	.macro PSHUFB_XMM xmm1 xmm2
-	XMM_NUM pshufb_opd1 \xmm1
-	XMM_NUM pshufb_opd2 \xmm2
-	PFX_OPD_SIZE
-	PFX_REX pshufb_opd1 pshufb_opd2
-	.byte 0x0f, 0x38, 0x00
-	MODRM 0xc0 pshufb_opd1 pshufb_opd2
-	.endm
-
-	.macro PCLMULQDQ imm8 xmm1 xmm2
-	XMM_NUM clmul_opd1 \xmm1
-	XMM_NUM clmul_opd2 \xmm2
-	PFX_OPD_SIZE
-	PFX_REX clmul_opd1 clmul_opd2
-	.byte 0x0f, 0x3a, 0x44
-	MODRM 0xc0 clmul_opd1 clmul_opd2
-	.byte \imm8
-	.endm
-
-	.macro PEXTRD imm8 xmm gpr
-	R32_NUM extrd_opd1 \gpr
-	XMM_NUM extrd_opd2 \xmm
-	PFX_OPD_SIZE
-	PFX_REX extrd_opd1 extrd_opd2
-	.byte 0x0f, 0x3a, 0x16
-	MODRM 0xc0 extrd_opd1 extrd_opd2
-	.byte \imm8
-	.endm
-
-	.macro MOVQ_R64_XMM opd1 opd2
-	TYPE movq_r64_xmm_opd1_type \opd1
-	.if movq_r64_xmm_opd1_type == TYPE_XMM
-	XMM_NUM movq_r64_xmm_opd1 \opd1
-	R64_NUM movq_r64_xmm_opd2 \opd2
-	.else
-	R64_NUM movq_r64_xmm_opd1 \opd1
-	XMM_NUM movq_r64_xmm_opd2 \opd2
-	.endif
-	PFX_OPD_SIZE
-	PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
-	.if movq_r64_xmm_opd1_type == TYPE_XMM
-	.byte 0x0f, 0x7e
-	.else
-	.byte 0x0f, 0x6e
-	.endif
-	MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
-	.endm
-
-#define	CTX	%rdi	/* arg1 */
-#define BUF	%rsi	/* arg2 */
-#define CNT	%rdx	/* arg3 */
-
-#define	REG_A	%ecx
-#define	REG_B	%esi
-#define	REG_C	%edi
-#define	REG_D	%eax
-#define	REG_E	%edx
-#define	REG_TB	%ebx
-#define	REG_TA	%r12d
-#define	REG_RA	%rcx
-#define	REG_RB	%rsi
-#define	REG_RC	%rdi
-#define	REG_RD	%rax
-#define	REG_RE	%rdx
-#define	REG_RTA	%r12
-#define	REG_RTB	%rbx
-#define	REG_T1	%ebp
-#define	xmm_mov	vmovups
-#define	avx2_zeroupper	vzeroupper
-#define	RND_F1	1
-#define	RND_F2	2
-#define	RND_F3	3
-
-.macro REGALLOC
-	.set A, REG_A
-	.set B, REG_B
-	.set C, REG_C
-	.set D, REG_D
-	.set E, REG_E
-	.set TB, REG_TB
-	.set TA, REG_TA
-
-	.set RA, REG_RA
-	.set RB, REG_RB
-	.set RC, REG_RC
-	.set RD, REG_RD
-	.set RE, REG_RE
-
-	.set RTA, REG_RTA
-	.set RTB, REG_RTB
-
-	.set T1, REG_T1
-.endm
-
-#define K_BASE		%r8
-#define HASH_PTR	%r9
-#define BUFFER_PTR	%r10
-#define BUFFER_PTR2	%r13
-#define BUFFER_END	%r11
-
-#define PRECALC_BUF	%r14
-#define WK_BUF		%r15
-
-#define W_TMP		%xmm0
-#define WY_TMP		%ymm0
-#define WY_TMP2		%ymm9
-
-# AVX2 variables
-#define WY0		%ymm3
-#define WY4		%ymm5
-#define WY08		%ymm7
-#define WY12		%ymm8
-#define WY16		%ymm12
-#define WY20		%ymm13
-#define WY24		%ymm14
-#define WY28		%ymm15
-
-#define YMM_SHUFB_BSWAP	%ymm10
-
-/*
- * Keep 2 iterations precalculated at a time:
- *    - 80 DWORDs per iteration * 2
- */
-#define W_SIZE		(80*2*2 +16)
-
-#define WK(t)	((((t) % 80) / 4)*32 + ( (t) % 4)*4 + ((t)/80)*16 )(WK_BUF)
-#define PRECALC_WK(t)	((t)*2*2)(PRECALC_BUF)
-
-
-.macro UPDATE_HASH  hash, val
-	add	\hash, \val
-	mov	\val, \hash
-.endm
-
-.macro PRECALC_RESET_WY
-	.set WY_00, WY0
-	.set WY_04, WY4
-	.set WY_08, WY08
-	.set WY_12, WY12
-	.set WY_16, WY16
-	.set WY_20, WY20
-	.set WY_24, WY24
-	.set WY_28, WY28
-	.set WY_32, WY_00
-.endm
-
-.macro PRECALC_ROTATE_WY
-	/* Rotate macros */
-	.set WY_32, WY_28
-	.set WY_28, WY_24
-	.set WY_24, WY_20
-	.set WY_20, WY_16
-	.set WY_16, WY_12
-	.set WY_12, WY_08
-	.set WY_08, WY_04
-	.set WY_04, WY_00
-	.set WY_00, WY_32
-
-	/* Define register aliases */
-	.set WY, WY_00
-	.set WY_minus_04, WY_04
-	.set WY_minus_08, WY_08
-	.set WY_minus_12, WY_12
-	.set WY_minus_16, WY_16
-	.set WY_minus_20, WY_20
-	.set WY_minus_24, WY_24
-	.set WY_minus_28, WY_28
-	.set WY_minus_32, WY
-.endm
-
-.macro PRECALC_00_15
-	.if (i == 0) # Initialize and rotate registers
-		PRECALC_RESET_WY
-		PRECALC_ROTATE_WY
-	.endif
-
-	/* message scheduling pre-compute for rounds 0-15 */
-	.if   ((i & 7) == 0)
-		/*
-		 * blended AVX2 and ALU instruction scheduling
-		 * 1 vector iteration per 8 rounds
-		 */
-		vmovdqu ((i * 2) + PRECALC_OFFSET)(BUFFER_PTR), W_TMP
-	.elseif ((i & 7) == 1)
-		vinsertf128 $1, (((i-1) * 2)+PRECALC_OFFSET)(BUFFER_PTR2),\
-			 WY_TMP, WY_TMP
-	.elseif ((i & 7) == 2)
-		vpshufb YMM_SHUFB_BSWAP, WY_TMP, WY
-	.elseif ((i & 7) == 4)
-		vpaddd  K_XMM(K_BASE), WY, WY_TMP
-	.elseif ((i & 7) == 7)
-		vmovdqu  WY_TMP, PRECALC_WK(i&~7)
-
-		PRECALC_ROTATE_WY
-	.endif
-.endm
-
-.macro PRECALC_16_31
-	/*
-	 * message scheduling pre-compute for rounds 16-31
-	 * calculating last 32 w[i] values in 8 XMM registers
-	 * pre-calculate K+w[i] values and store to mem
-	 * for later load by ALU add instruction
-	 *
-	 * "brute force" vectorization for rounds 16-31 only
-	 * due to w[i]->w[i-3] dependency
-	 */
-	.if   ((i & 7) == 0)
-		/*
-		 * blended AVX2 and ALU instruction scheduling
-		 * 1 vector iteration per 8 rounds
-		 */
-		/* w[i-14] */
-		vpalignr	$8, WY_minus_16, WY_minus_12, WY
-		vpsrldq	$4, WY_minus_04, WY_TMP               /* w[i-3] */
-	.elseif ((i & 7) == 1)
-		vpxor	WY_minus_08, WY, WY
-		vpxor	WY_minus_16, WY_TMP, WY_TMP
-	.elseif ((i & 7) == 2)
-		vpxor	WY_TMP, WY, WY
-		vpslldq	$12, WY, WY_TMP2
-	.elseif ((i & 7) == 3)
-		vpslld	$1, WY, WY_TMP
-		vpsrld	$31, WY, WY
-	.elseif ((i & 7) == 4)
-		vpor	WY, WY_TMP, WY_TMP
-		vpslld	$2, WY_TMP2, WY
-	.elseif ((i & 7) == 5)
-		vpsrld	$30, WY_TMP2, WY_TMP2
-		vpxor	WY, WY_TMP, WY_TMP
-	.elseif ((i & 7) == 7)
-		vpxor	WY_TMP2, WY_TMP, WY
-		vpaddd	K_XMM(K_BASE), WY, WY_TMP
-		vmovdqu	WY_TMP, PRECALC_WK(i&~7)
-
-		PRECALC_ROTATE_WY
-	.endif
-.endm
-
-.macro PRECALC_32_79
-	/*
-	 * in SHA-1 specification:
-	 * w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
-	 * instead we do equal:
-	 * w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
-	 * allows more efficient vectorization
-	 * since w[i]=>w[i-3] dependency is broken
-	 */
-
-	.if   ((i & 7) == 0)
-	/*
-	 * blended AVX2 and ALU instruction scheduling
-	 * 1 vector iteration per 8 rounds
-	 */
-		vpalignr	$8, WY_minus_08, WY_minus_04, WY_TMP
-	.elseif ((i & 7) == 1)
-		/* W is W_minus_32 before xor */
-		vpxor	WY_minus_28, WY, WY
-	.elseif ((i & 7) == 2)
-		vpxor	WY_minus_16, WY_TMP, WY_TMP
-	.elseif ((i & 7) == 3)
-		vpxor	WY_TMP, WY, WY
-	.elseif ((i & 7) == 4)
-		vpslld	$2, WY, WY_TMP
-	.elseif ((i & 7) == 5)
-		vpsrld	$30, WY, WY
-		vpor	WY, WY_TMP, WY
-	.elseif ((i & 7) == 7)
-		vpaddd	K_XMM(K_BASE), WY, WY_TMP
-		vmovdqu	WY_TMP, PRECALC_WK(i&~7)
-
-		PRECALC_ROTATE_WY
-	.endif
-.endm
-
-.macro PRECALC r, s
-	.set i, \r
-
-	.if (i < 40)
-		.set K_XMM, 32*0
-	.elseif (i < 80)
-		.set K_XMM, 32*1
-	.elseif (i < 120)
-		.set K_XMM, 32*2
-	.else
-		.set K_XMM, 32*3
-	.endif
-
-	.if (i<32)
-		PRECALC_00_15	\s
-	.elseif (i<64)
-		PRECALC_16_31	\s
-	.elseif (i < 160)
-		PRECALC_32_79	\s
-	.endif
-.endm
-
-.macro ROTATE_STATE
-	.set T_REG, E
-	.set E, D
-	.set D, C
-	.set C, B
-	.set B, TB
-	.set TB, A
-	.set A, T_REG
-
-	.set T_REG, RE
-	.set RE, RD
-	.set RD, RC
-	.set RC, RB
-	.set RB, RTB
-	.set RTB, RA
-	.set RA, T_REG
-.endm
-
-/* Macro relies on saved ROUND_Fx */
-
-.macro RND_FUN f, r
-	.if (\f == RND_F1)
-		ROUND_F1	\r
-	.elseif (\f == RND_F2)
-		ROUND_F2	\r
-	.elseif (\f == RND_F3)
-		ROUND_F3	\r
-	.endif
-.endm
-
-.macro RR r
-	.set round_id, (\r % 80)
-
-	.if (round_id == 0)        /* Precalculate F for first round */
-		.set ROUND_FUNC, RND_F1
-		mov	B, TB
-
-		rorx	$(32-30), B, B    /* b>>>2 */
-		andn	D, TB, T1
-		and	C, TB
-		xor	T1, TB
-	.endif
-
-	RND_FUN ROUND_FUNC, \r
-	ROTATE_STATE
-
-	.if   (round_id == 18)
-		.set ROUND_FUNC, RND_F2
-	.elseif (round_id == 38)
-		.set ROUND_FUNC, RND_F3
-	.elseif (round_id == 58)
-		.set ROUND_FUNC, RND_F2
-	.endif
-
-	.set round_id, ( (\r+1) % 80)
-
-	RND_FUN ROUND_FUNC, (\r+1)
-	ROTATE_STATE
-.endm
-
-.macro ROUND_F1 r
-	add	WK(\r), E
-
-	andn	C, A, T1			/* ~b&d */
-	lea	(RE,RTB), E		/* Add F from the previous round */
-
-	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
-	rorx	$(32-30),A, TB		/* b>>>2 for next round */
-
-	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
-
-	/*
-	 * Calculate F for the next round
-	 * (b & c) ^ andn[b, d]
-	 */
-	and	B, A			/* b&c */
-	xor	T1, A			/* F1 = (b&c) ^ (~b&d) */
-
-	lea	(RE,RTA), E		/* E += A >>> 5 */
-.endm
-
-.macro ROUND_F2 r
-	add	WK(\r), E
-	lea	(RE,RTB), E		/* Add F from the previous round */
-
-	/* Calculate F for the next round */
-	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
-	.if ((round_id) < 79)
-		rorx	$(32-30), A, TB	/* b>>>2 for next round */
-	.endif
-	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
-
-	.if ((round_id) < 79)
-		xor	B, A
-	.endif
-
-	add	TA, E			/* E += A >>> 5 */
-
-	.if ((round_id) < 79)
-		xor	C, A
-	.endif
-.endm
-
-.macro ROUND_F3 r
-	add	WK(\r), E
-	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
-
-	lea	(RE,RTB), E		/* Add F from the previous round */
-
-	mov	B, T1
-	or	A, T1
-
-	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
-	rorx	$(32-30), A, TB		/* b>>>2 for next round */
-
-	/* Calculate F for the next round
-	 * (b and c) or (d and (b or c))
-	 */
-	and	C, T1
-	and	B, A
-	or	T1, A
-
-	add	TA, E			/* E += A >>> 5 */
-
-.endm
-
-/*
- * macro implements 80 rounds of SHA-1, for multiple blocks with s/w pipelining
- */
-.macro SHA1_PIPELINED_MAIN_BODY
-
-	REGALLOC
-
-	mov	(HASH_PTR), A
-	mov	4(HASH_PTR), B
-	mov	8(HASH_PTR), C
-	mov	12(HASH_PTR), D
-	mov	16(HASH_PTR), E
-
-	mov	%rsp, PRECALC_BUF
-	lea	(2*4*80+32)(%rsp), WK_BUF
-
-	# Precalc WK for first 2 blocks
-	PRECALC_OFFSET = 0
-	.set i, 0
-	.rept    160
-		PRECALC i
-		.set i, i + 1
-	.endr
-	PRECALC_OFFSET = 128
-	xchg	WK_BUF, PRECALC_BUF
-
-	.align 32
-_loop:
-	/*
-	 * code loops through more than one block
-	 * we use K_BASE value as a signal of a last block,
-	 * it is set below by: cmovae BUFFER_PTR, K_BASE
-	 */
-	cmp	K_BASE, BUFFER_PTR
-	jne	_begin
-	.align 32
-	jmp	_end
-	.align 32
-_begin:
-
-	/*
-	 * Do first block
-	 * rounds: 0,2,4,6,8
-	 */
-	.set j, 0
-	.rept 5
-		RR	j
-		.set j, j+2
-	.endr
-
-	jmp _loop0
-_loop0:
-
-	/*
-	 * rounds:
-	 * 10,12,14,16,18
-	 * 20,22,24,26,28
-	 * 30,32,34,36,38
-	 * 40,42,44,46,48
-	 * 50,52,54,56,58
-	 */
-	.rept 25
-		RR	j
-		.set j, j+2
-	.endr
-
-	add	$(2*64), BUFFER_PTR       /* move to next odd-64-byte block */
-	cmp	BUFFER_END, BUFFER_PTR    /* is current block the last one? */
-	cmovae	K_BASE, BUFFER_PTR	/* signal the last iteration smartly */
-
-	/*
-	 * rounds
-	 * 60,62,64,66,68
-	 * 70,72,74,76,78
-	 */
-	.rept 10
-		RR	j
-		.set j, j+2
-	.endr
-
-	UPDATE_HASH	(HASH_PTR), A
-	UPDATE_HASH	4(HASH_PTR), TB
-	UPDATE_HASH	8(HASH_PTR), C
-	UPDATE_HASH	12(HASH_PTR), D
-	UPDATE_HASH	16(HASH_PTR), E
-
-	cmp	K_BASE, BUFFER_PTR	/* is current block the last one? */
-	je	_loop
-
-	mov	TB, B
-
-	/* Process second block */
-	/*
-	 * rounds
-	 *  0+80, 2+80, 4+80, 6+80, 8+80
-	 * 10+80,12+80,14+80,16+80,18+80
-	 */
-
-	.set j, 0
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	jmp	_loop1
-_loop1:
-	/*
-	 * rounds
-	 * 20+80,22+80,24+80,26+80,28+80
-	 * 30+80,32+80,34+80,36+80,38+80
-	 */
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	jmp	_loop2
-_loop2:
-
-	/*
-	 * rounds
-	 * 40+80,42+80,44+80,46+80,48+80
-	 * 50+80,52+80,54+80,56+80,58+80
-	 */
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	add	$(2*64), BUFFER_PTR2      /* move to next even-64-byte block */
-
-	cmp	BUFFER_END, BUFFER_PTR2   /* is current block the last one */
-	cmovae	K_BASE, BUFFER_PTR       /* signal the last iteration smartly */
-
-	jmp	_loop3
-_loop3:
-
-	/*
-	 * rounds
-	 * 60+80,62+80,64+80,66+80,68+80
-	 * 70+80,72+80,74+80,76+80,78+80
-	 */
-	.rept 10
-		RR	j+80
-		.set j, j+2
-	.endr
-
-	UPDATE_HASH	(HASH_PTR), A
-	UPDATE_HASH	4(HASH_PTR), TB
-	UPDATE_HASH	8(HASH_PTR), C
-	UPDATE_HASH	12(HASH_PTR), D
-	UPDATE_HASH	16(HASH_PTR), E
-
-	/* Reset state for AVX2 reg permutation */
-	mov	A, TA
-	mov	TB, A
-	mov	C, TB
-	mov	E, C
-	mov	D, B
-	mov	TA, D
-
-	REGALLOC
-
-	xchg	WK_BUF, PRECALC_BUF
-
-	jmp	_loop
-
-	.align 32
-	_end:
-
-.endm
-
-.section .rodata
-
-#define K1 0x5a827999
-#define K2 0x6ed9eba1
-#define K3 0x8f1bbcdc
-#define K4 0xca62c1d6
-
-.align 128
-K_XMM_AR:
-	.long K1, K1, K1, K1
-	.long K1, K1, K1, K1
-	.long K2, K2, K2, K2
-	.long K2, K2, K2, K2
-	.long K3, K3, K3, K3
-	.long K3, K3, K3, K3
-	.long K4, K4, K4, K4
-	.long K4, K4, K4, K4
-
-BSWAP_SHUFB_CTL:
-	.long 0x00010203
-	.long 0x04050607
-	.long 0x08090a0b
-	.long 0x0c0d0e0f
-	.long 0x00010203
-	.long 0x04050607
-	.long 0x08090a0b
-	.long 0x0c0d0e0f
-
-# void sha1_transform(int32_t *hash, const char* input, size_t num_blocks) ;
-        .text
-	ENTRY(sha1_transform)
-
-	push	%rbx
-	push	%rbp
-	push	%r12
-	push	%r13
-	push	%r14
-	push	%r15
-
-	RESERVE_STACK  = (W_SIZE*4 + 8+24)
-
-	/* Align stack */
-	mov	%rsp, %rbx
-	and	$~(0x20-1), %rsp
-	push	%rbx
-	sub	$RESERVE_STACK, %rsp
-
-	avx2_zeroupper
-
-	lea	K_XMM_AR(%rip), K_BASE
-
-	mov	CTX, HASH_PTR
-	mov	BUF, BUFFER_PTR
-	lea	64(BUF), BUFFER_PTR2
-
-	shl	$6, CNT			/* mul by 64 */
-	add	BUF, CNT
-	add	$64, CNT
-	mov	CNT, BUFFER_END
-
-	cmp	BUFFER_END, BUFFER_PTR2
-	cmovae	K_BASE, BUFFER_PTR2
-
-	xmm_mov	BSWAP_SHUFB_CTL(%rip), YMM_SHUFB_BSWAP
-
-	SHA1_PIPELINED_MAIN_BODY
-
-	avx2_zeroupper
-
-	add	$RESERVE_STACK, %rsp
-	pop	%rsp
-
-	pop	%r15
-	pop	%r14
-	pop	%r13
-	pop	%r12
-	pop	%rbp
-	pop	%rbx
-
-	ret
-
-	ENDPROC(sha1_transform)
-#endif

pkg/crypto/sha1/sha1_sse3_amd64.asm

@@ -1,579 +0,0 @@
-;---------------------
-;   https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
-;
-; License information:
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;  This implementation notably advances the performance of SHA-1 algorithm compared to existing
-;  implementations. We are encouraging all projects utilizing SHA-1 to integrate this new fast
-;  implementation and are ready to help if issues or concerns arise (you are welcome to leave
-;  a comment or write an email to the authors). It is provided 'as is' and free for either
-;  commercial or non-commercial use.
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;
-;   This code implements two interfaces of SHA-1 update function: 1) working on a single
-;   64-byte block and 2) working on a buffer of multiple 64-bit blocks. Multiple blocks
-;   version of code is software pipelined and faster overall, it is a default. Assemble
-;   with -DINTEL_SHA1_SINGLEBLOCK to select single 64-byte block function interface.
-;
-;   C++ prototypes of implemented functions are below:
-;
-;   #ifndef INTEL_SHA1_SINGLEBLOCK
-;      // Updates 20-byte SHA-1 record in 'hash' for 'num_blocks' consequtive 64-byte blocks
-;      extern "C" void sha1_update_intel(int *hash, const char* input, size_t num_blocks );
-;   #else
-;      // Updates 20-byte SHA-1 record in 'hash' for one 64-byte block pointed by 'input'
-;      extern "C" void sha1_update_intel(int *hash, const char* input);
-;   #endif
-;
-;   Function name 'sha1_update_intel' can be changed in the source or via macro:
-;     -DINTEL_SHA1_UPDATE_FUNCNAME=my_sha1_update_func_name
-;
-;   It implements both UNIX(default) and Windows ABIs, use -DWIN_ABI on Windows
-;
-;   Code checks CPU for SSSE3 support via CPUID feature flag (CPUID.1.ECX.SSSE3[bit 9]==1),
-;   and performs dispatch. Since in most cases the functionality on non-SSSE3 supporting CPUs
-;   is also required, the default (e.g. one being replaced) function can be provided for
-;   dispatch on such CPUs, the name of old function can be changed in the source or via macro:
-;      -DINTEL_SHA1_UPDATE_DEFAULT_DISPATCH=default_sha1_update_function_name
-;
-;   Authors: Maxim Locktyukhin and Ronen Zohar at Intel.com
-;
-
-%ifndef INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
- ;; can be replaced with a default SHA-1 update function name
-%define INTEL_SHA1_UPDATE_DEFAULT_DISPATCH  sha1_intel_non_ssse3_cpu_stub_
-%else
-extern  INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
-%endif
-
-;; provide alternative SHA-1 update function's name here
-%ifndef INTEL_SHA1_UPDATE_FUNCNAME
-%define INTEL_SHA1_UPDATE_FUNCNAME     sha1_update_intel
-%endif
-
-global INTEL_SHA1_UPDATE_FUNCNAME
-
-
-%ifndef INTEL_SHA1_SINGLEBLOCK
-%assign multiblock 1
-%else
-%assign multiblock 0
-%endif
-
-
-bits 64
-default rel
-
-%ifdef WIN_ABI
- %xdefine arg1 rcx
- %xdefine arg2 rdx
- %xdefine arg3 r8
-%else
- %xdefine arg1 rdi
- %xdefine arg2 rsi
- %xdefine arg3 rdx
-%endif
-
-%xdefine ctx arg1
-%xdefine buf arg2
-%xdefine cnt arg3
-
-%macro REGALLOC 0
- %xdefine A ecx
- %xdefine B esi
- %xdefine C edi
- %xdefine D ebp
- %xdefine E edx
-
- %xdefine T1 eax
- %xdefine T2 ebx
-%endmacro
-
-%xdefine K_BASE     r8
-%xdefine HASH_PTR   r9
-%xdefine BUFFER_PTR r10
-%xdefine BUFFER_END r11
-
-%xdefine W_TMP  xmm0
-%xdefine W_TMP2 xmm9
-
-%xdefine W0  xmm1
-%xdefine W4  xmm2
-%xdefine W8  xmm3
-%xdefine W12 xmm4
-%xdefine W16 xmm5
-%xdefine W20 xmm6
-%xdefine W24 xmm7
-%xdefine W28 xmm8
-
-%xdefine XMM_SHUFB_BSWAP xmm10
-
-;; we keep window of 64 w[i]+K pre-calculated values in a circular buffer
-%xdefine WK(t) (rsp + (t & 15)*4)
-
-;------------------------------------------------------------------------------
-;
-; macro implements SHA-1 function's body for single or several 64-byte blocks
-; first param: function's name
-; second param: =0 - function implements single 64-byte block hash
-;               =1 - function implements multiple64-byte blocks hash
-;                    3rd function's argument is a number, greater 0, of 64-byte blocks to calc hash for
-;
-%macro  SHA1_VECTOR_ASM  2
-align 4096
-%1:
- push rbx
- push rbp
-
- %ifdef WIN_ABI
- push rdi
- push rsi
-
- %xdefine stack_size (16*4 + 16*5 + 8)
- %else
- %xdefine stack_size (16*4 + 8)
- %endif
-
- sub     rsp, stack_size
-
- %ifdef WIN_ABI
- %xdefine xmm_save_base (rsp + 16*4)
-
- xmm_mov [xmm_save_base + 0*16], xmm6
- xmm_mov [xmm_save_base + 1*16], xmm7
- xmm_mov [xmm_save_base + 2*16], xmm8
- xmm_mov [xmm_save_base + 3*16], xmm9
- xmm_mov [xmm_save_base + 4*16], xmm10
- %endif
-
- mov     HASH_PTR, ctx
- mov     BUFFER_PTR, buf
-
- %if (%2 == 1)
- shl     cnt, 6           ;; mul by 64
- add     cnt, buf
- mov     BUFFER_END, cnt
- %endif
-
- lea     K_BASE, [K_XMM_AR]
- xmm_mov XMM_SHUFB_BSWAP, [bswap_shufb_ctl]
-
- SHA1_PIPELINED_MAIN_BODY %2
-
- %ifdef WIN_ABI
- xmm_mov xmm6, [xmm_save_base + 0*16]
- xmm_mov xmm7, [xmm_save_base + 1*16]
- xmm_mov xmm8, [xmm_save_base + 2*16]
- xmm_mov xmm9, [xmm_save_base + 3*16]
- xmm_mov xmm10,[xmm_save_base + 4*16]
- %endif
-
- add rsp, stack_size
-
- %ifdef WIN_ABI
- pop rsi
- pop rdi
- %endif
-
- pop rbp
- pop rbx
-
- ret
-%endmacro
-
-;--------------------------------------------
-; macro implements 80 rounds of SHA-1, for one 64-byte block or multiple blocks with s/w pipelining
-; macro param: =0 - process single 64-byte block
-;              =1 - multiple blocks
-;
-%macro SHA1_PIPELINED_MAIN_BODY 1
-
- REGALLOC
-
- mov A, [HASH_PTR   ]
- mov B, [HASH_PTR+ 4]
- mov C, [HASH_PTR+ 8]
- mov D, [HASH_PTR+12]
-
- mov E, [HASH_PTR+16]
-
- %assign i 0
- %rep    W_PRECALC_AHEAD
- W_PRECALC i
- %assign i i+1
- %endrep
-
- %xdefine F F1
-
- %if (%1 == 1)                         ;; code loops through more than one block
- %%_loop:
- cmp BUFFER_PTR, K_BASE          ;; we use K_BASE value as a signal of a last block,
- jne %%_begin                    ;; it is set below by: cmovae BUFFER_PTR, K_BASE
- jmp %%_end
-
- align 32
- %%_begin:
- %endif
- RR A,B,C,D,E,0
- RR D,E,A,B,C,2
- RR B,C,D,E,A,4
- RR E,A,B,C,D,6
- RR C,D,E,A,B,8
-
- RR A,B,C,D,E,10
- RR D,E,A,B,C,12
- RR B,C,D,E,A,14
- RR E,A,B,C,D,16
- RR C,D,E,A,B,18
-
- %xdefine F F2
-
- RR A,B,C,D,E,20
- RR D,E,A,B,C,22
- RR B,C,D,E,A,24
- RR E,A,B,C,D,26
- RR C,D,E,A,B,28
-
- RR A,B,C,D,E,30
- RR D,E,A,B,C,32
- RR B,C,D,E,A,34
- RR E,A,B,C,D,36
- RR C,D,E,A,B,38
-
- %xdefine F F3
-
- RR A,B,C,D,E,40
- RR D,E,A,B,C,42
- RR B,C,D,E,A,44
- RR E,A,B,C,D,46
- RR C,D,E,A,B,48
-
- RR A,B,C,D,E,50
- RR D,E,A,B,C,52
- RR B,C,D,E,A,54
- RR E,A,B,C,D,56
- RR C,D,E,A,B,58
-
- %xdefine F F4
-
- %if (%1 == 1)                         ;; if code loops through more than one block
- add   BUFFER_PTR, 64            ;; move to next 64-byte block
- cmp   BUFFER_PTR, BUFFER_END    ;; check if current block is the last one
- cmovae BUFFER_PTR, K_BASE       ;; smart way to signal the last iteration
- %else
- %xdefine W_NO_TAIL_PRECALC 1    ;; no software pipelining for single block interface
- %endif
-
- RR A,B,C,D,E,60
- RR D,E,A,B,C,62
- RR B,C,D,E,A,64
- RR E,A,B,C,D,66
- RR C,D,E,A,B,68
-
- RR A,B,C,D,E,70
- RR D,E,A,B,C,72
- RR B,C,D,E,A,74
- RR E,A,B,C,D,76
- RR C,D,E,A,B,78
-
- UPDATE_HASH [HASH_PTR   ],A
- UPDATE_HASH [HASH_PTR+ 4],B
- UPDATE_HASH [HASH_PTR+ 8],C
- UPDATE_HASH [HASH_PTR+12],D
- UPDATE_HASH [HASH_PTR+16],E
-
- %if (%1 == 1)
- jmp %%_loop
-
- align 32
- %%_end:
- %endif
-
-
- %xdefine W_NO_TAIL_PRECALC 0
- %xdefine F %error
-
-%endmacro
-
-
-%macro F1 3
- mov T1,%2
- xor T1,%3
- and T1,%1
- xor T1,%3
-%endmacro
-
-%macro F2 3
- mov T1,%3
- xor T1,%2
- xor T1,%1
-%endmacro
-
-%macro F3 3
- mov T1,%2
- mov T2,%1
- or  T1,%1
- and T2,%2
- and T1,%3
- or  T1,T2
-%endmacro
-
-%define F4 F2
-
-%macro UPDATE_HASH 2
- add %2, %1
- mov %1, %2
-%endmacro
-
-
-%macro W_PRECALC 1
- %xdefine i (%1)
-
- %if (i < 20)
- %xdefine K_XMM  0
- %elif (i < 40)
- %xdefine K_XMM  16
- %elif (i < 60)
- %xdefine K_XMM  32
- %else
- %xdefine K_XMM  48
- %endif
-
- %if (i<16 || (i>=80 && i<(80 + W_PRECALC_AHEAD)))
-
- %if (W_NO_TAIL_PRECALC == 0)
-
- %xdefine i ((%1) % 80)        ;; pre-compute for the next iteration
-
- %if (i == 0)
- W_PRECALC_RESET
- %endif
-
-
- W_PRECALC_00_15
- %endif
-
- %elif (i < 32)
- W_PRECALC_16_31
- %elif (i < 80)   ;; rounds 32-79
- W_PRECALC_32_79
- %endif
-%endmacro
-
-%macro W_PRECALC_RESET 0
- %xdefine    W             W0
- %xdefine    W_minus_04    W4
- %xdefine    W_minus_08    W8
- %xdefine    W_minus_12    W12
- %xdefine    W_minus_16    W16
- %xdefine    W_minus_20    W20
- %xdefine    W_minus_24    W24
- %xdefine    W_minus_28    W28
- %xdefine    W_minus_32    W
-%endmacro
-
-%macro W_PRECALC_ROTATE 0
- %xdefine    W_minus_32    W_minus_28
- %xdefine    W_minus_28    W_minus_24
- %xdefine    W_minus_24    W_minus_20
- %xdefine    W_minus_20    W_minus_16
- %xdefine    W_minus_16    W_minus_12
- %xdefine    W_minus_12    W_minus_08
- %xdefine    W_minus_08    W_minus_04
- %xdefine    W_minus_04    W
- %xdefine    W             W_minus_32
-%endmacro
-
-%xdefine W_PRECALC_AHEAD   16
-%xdefine W_NO_TAIL_PRECALC 0
-
-
-%xdefine xmm_mov            movdqa
-
-%macro W_PRECALC_00_15 0
- ;; message scheduling pre-compute for rounds 0-15
- %if ((i & 3) == 0)       ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
- movdqu W_TMP, [BUFFER_PTR + (i * 4)]
- %elif ((i & 3) == 1)
- pshufb W_TMP, XMM_SHUFB_BSWAP
- movdqa W, W_TMP
- %elif ((i & 3) == 2)
- paddd  W_TMP, [K_BASE]
- %elif ((i & 3) == 3)
- movdqa  [WK(i&~3)], W_TMP
-
- W_PRECALC_ROTATE
- %endif
-%endmacro
-
-%macro W_PRECALC_16_31 0
- ;; message scheduling pre-compute for rounds 16-31
- ;; calculating last 32 w[i] values in 8 XMM registers
- ;; pre-calculate K+w[i] values and store to mem, for later load by ALU add instruction
- ;;
- ;; "brute force" vectorization for rounds 16-31 only due to w[i]->w[i-3] dependency
- ;;
- %if ((i & 3) == 0)    ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
- movdqa  W, W_minus_12
- palignr W, W_minus_16, 8       ;; w[i-14]
- movdqa  W_TMP, W_minus_04
- psrldq  W_TMP, 4               ;; w[i-3]
- pxor    W, W_minus_08
- %elif ((i & 3) == 1)
- pxor    W_TMP, W_minus_16
- pxor    W, W_TMP
- movdqa  W_TMP2, W
- movdqa  W_TMP, W
- pslldq  W_TMP2, 12
- %elif ((i & 3) == 2)
- psrld   W, 31
- pslld   W_TMP, 1
- por     W_TMP, W
- movdqa  W, W_TMP2
- psrld   W_TMP2, 30
- pslld   W, 2
- %elif ((i & 3) == 3)
- pxor    W_TMP, W
- pxor    W_TMP, W_TMP2
- movdqa  W, W_TMP
- paddd   W_TMP, [K_BASE + K_XMM]
- movdqa  [WK(i&~3)],W_TMP
-
- W_PRECALC_ROTATE
- %endif
-%endmacro
-
-%macro W_PRECALC_32_79 0
- ;; in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
- ;; instead we do equal:    w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
- ;; allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
- ;;
- %if ((i & 3) == 0)    ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
- movdqa  W_TMP, W_minus_04
- pxor    W, W_minus_28         ;; W is W_minus_32 before xor
- palignr W_TMP, W_minus_08, 8
- %elif ((i & 3) == 1)
- pxor    W, W_minus_16
- pxor    W, W_TMP
- movdqa  W_TMP, W
- %elif ((i & 3) == 2)
- psrld   W, 30
- pslld   W_TMP, 2
- por     W_TMP, W
- %elif ((i & 3) == 3)
- movdqa  W, W_TMP
- paddd   W_TMP, [K_BASE + K_XMM]
- movdqa  [WK(i&~3)],W_TMP
-
- W_PRECALC_ROTATE
- %endif
-%endmacro
-
-%macro RR 6             ;; RR does two rounds of SHA-1 back to back with W pre-calculation
-
- ;;     TEMP = A
- ;;     A = F( i, B, C, D ) + E + ROTATE_LEFT( A, 5 ) + W[i] + K(i)
- ;;     C = ROTATE_LEFT( B, 30 )
- ;;     D = C
- ;;     E = D
- ;;     B = TEMP
-
- W_PRECALC (%6 + W_PRECALC_AHEAD)
- F    %2, %3, %4     ;; F returns result in T1
- add  %5, [WK(%6)]
- rol  %2, 30
- mov  T2, %1
- add  %4, [WK(%6 + 1)]
- rol  T2, 5
- add  %5, T1
-
- W_PRECALC (%6 + W_PRECALC_AHEAD + 1)
- add  T2, %5
- mov  %5, T2
- rol  T2, 5
- add  %4, T2
- F    %1, %2, %3    ;; F returns result in T1
- add  %4, T1
- rol  %1, 30
-
-;; write:  %1, %2
-;; rotate: %1<=%4, %2<=%5, %3<=%1, %4<=%2, %5<=%3
-%endmacro
-
-
-
-;;----------------------
-section .data align=128
-
-%xdefine K1 0x5a827999
-%xdefine K2 0x6ed9eba1
-%xdefine K3 0x8f1bbcdc
-%xdefine K4 0xca62c1d6
-
-align 128
-K_XMM_AR:
- DD K1, K1, K1, K1
- DD K2, K2, K2, K2
- DD K3, K3, K3, K3
- DD K4, K4, K4, K4
-
-align 16
-bswap_shufb_ctl:
- DD 00010203h
- DD 04050607h
- DD 08090a0bh
- DD 0c0d0e0fh
-
-;; dispatch pointer, points to the init routine for the first invocation
-sha1_update_intel_dispatched:
- DQ  sha1_update_intel_init_
-
-;;----------------------
-section .text align=4096
-
-SHA1_VECTOR_ASM     sha1_update_intel_ssse3_, multiblock
-
-align 32
-sha1_update_intel_init_:       ;; we get here with the first time invocation
- call    sha1_update_intel_dispacth_init_
-INTEL_SHA1_UPDATE_FUNCNAME:    ;; we get here after init
- jmp     qword [sha1_update_intel_dispatched]
-
-;; CPUID feature flag based dispatch
-sha1_update_intel_dispacth_init_:
- push    rax
- push    rbx
- push    rcx
- push    rdx
- push    rsi
-
- lea     rsi, [INTEL_SHA1_UPDATE_DEFAULT_DISPATCH]
-
- mov     eax, 1
- cpuid
-
- test    ecx, 0200h          ;; SSSE3 support, CPUID.1.ECX[bit 9]
- jz      _done
-
- lea     rsi, [sha1_update_intel_ssse3_]
-
-_done:
- mov     [sha1_update_intel_dispatched], rsi
-
- pop     rsi
- pop     rdx
- pop     rcx
- pop     rbx
- pop     rax
- ret
-
-;;----------------------
-;; in the case a default SHA-1 update function implementation was not provided
-;; and code was invoked on a non-SSSE3 supporting CPU, dispatch handles this
-;; failure in a safest way - jumps to the stub function with UD2 instruction below
-sha1_intel_non_ssse3_cpu_stub_:
- ud2     ;; in the case no default SHA-1 was provided non-SSSE3 CPUs safely fail here
- ret
-
-; END
-;----------------------

pkg/crypto/sha1/sha1_test.go

@@ -1,154 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file of
-// Golang project:
-//    https://github.com/golang/go/blob/master/LICENSE
-
-// Using this part of Minio codebase under the license
-// Apache License Version 2.0 with modifications
-
-// SHA1 hash algorithm.  See RFC 3174.
-
-package sha1
-
-import (
-	"crypto/rand"
-	"fmt"
-	"io"
-	"testing"
-)
-
-type sha1Test struct {
-	out string
-	in  string
-}
-
-var golden = []sha1Test{
-	{"da39a3ee5e6b4b0d3255bfef95601890afd80709", ""},
-	{"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a"},
-	{"da23614e02469a0d7c7bd1bdab5c9c474b1904dc", "ab"},
-	{"a9993e364706816aba3e25717850c26c9cd0d89d", "abc"},
-	{"81fe8bfe87576c3ecb22426f8e57847382917acf", "abcd"},
-	{"03de6c570bfe24bfc328ccd7ca46b76eadaf4334", "abcde"},
-	{"1f8ac10f23c5b5bc1167bda84b833e5c057a77d2", "abcdef"},
-	{"2fb5e13419fc89246865e7a324f476ec624e8740", "abcdefg"},
-	{"425af12a0743502b322e93a015bcf868e324d56a", "abcdefgh"},
-	{"c63b19f1e4c8b5f76b25c49b8b87f57d8e4872a1", "abcdefghi"},
-	{"d68c19a0a345b7eab78d5e11e991c026ec60db63", "abcdefghij"},
-	{"ebf81ddcbe5bf13aaabdc4d65354fdf2044f38a7", "Discard medicine more than two years old."},
-	{"e5dea09392dd886ca63531aaa00571dc07554bb6", "He who has a shady past knows that nice guys finish last."},
-	{"45988f7234467b94e3e9494434c96ee3609d8f8f", "I wouldn't marry him with a ten foot pole."},
-	{"55dee037eb7460d5a692d1ce11330b260e40c988", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
-	{"b7bc5fb91080c7de6b582ea281f8a396d7c0aee8", "The days of the digital watch are numbered.  -Tom Stoppard"},
-	{"c3aed9358f7c77f523afe86135f06b95b3999797", "Nepal premier won't resign."},
-	{"6e29d302bf6e3a5e4305ff318d983197d6906bb9", "For every action there is an equal and opposite government program."},
-	{"597f6a540010f94c15d71806a99a2c8710e747bd", "His money is twice tainted: 'taint yours and 'taint mine."},
-	{"6859733b2590a8a091cecf50086febc5ceef1e80", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
-	{"514b2630ec089b8aee18795fc0cf1f4860cdacad", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
-	{"c5ca0d4a7b6676fc7aa72caa41cc3d5df567ed69", "size:  a.out:  bad magic"},
-	{"74c51fa9a04eadc8c1bbeaa7fc442f834b90a00a", "The major problem is with sendmail.  -Mark Horton"},
-	{"0b4c4ce5f52c3ad2821852a8dc00217fa18b8b66", "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
-	{"3ae7937dd790315beb0f48330e8642237c61550a", "If the enemy is within range, then so are you."},
-	{"410a2b296df92b9a47412b13281df8f830a9f44b", "It's well we cannot hear the screams/That we create in others' dreams."},
-	{"841e7c85ca1adcddbdd0187f1289acb5c642f7f5", "You remind me of a TV show, but that's all right: I watch it anyway."},
-	{"163173b825d03b952601376b25212df66763e1db", "C is as portable as Stonehedge!!"},
-	{"32b0377f2687eb88e22106f133c586ab314d5279", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
-	{"0885aaf99b569542fd165fa44e322718f4a984e0", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
-	{"6627d6904d71420b0bf3886ab629623538689f45", "How can you write a big system without C++?  -Paul Glick"},
-}
-
-func TestGolden(t *testing.T) {
-	for i := 0; i < len(golden); i++ {
-		g := golden[i]
-		s := fmt.Sprintf("%x", Sum([]byte(g.in)))
-		if s != g.out {
-			t.Fatalf("Sum function: sha1(%s) = %s want %s", g.in, s, g.out)
-		}
-		c := New()
-		for j := 0; j < 3; j++ {
-			if j < 2 {
-				io.WriteString(c, g.in)
-			} else {
-				io.WriteString(c, g.in[0:len(g.in)/2])
-				c.Sum(nil)
-				io.WriteString(c, g.in[len(g.in)/2:])
-			}
-			s := fmt.Sprintf("%x", c.Sum(nil))
-			if s != g.out {
-				t.Fatalf("sha1[%d](%s) = %s want %s", j, g.in, s, g.out)
-			}
-			c.Reset()
-		}
-	}
-}
-
-func TestSize(t *testing.T) {
-	c := New()
-	if got := c.Size(); got != Size {
-		t.Errorf("Size = %d; want %d", got, Size)
-	}
-}
-
-func TestBlockSize(t *testing.T) {
-	c := New()
-	if got := c.BlockSize(); got != BlockSize {
-		t.Errorf("BlockSize = %d; want %d", got, BlockSize)
-	}
-}
-
-// Tests that blockGeneric (pure Go) and block (in assembly for amd64, 386, arm) match.
-func TestBlockGeneric(t *testing.T) {
-	gen, asm := New().(*digest), New().(*digest)
-	buf := make([]byte, BlockSize*20) // arbitrary factor
-	rand.Read(buf)
-	blockGeneric(gen, buf)
-	block(asm, buf)
-	if *gen != *asm {
-		t.Error("block and blockGeneric resulted in different states")
-	}
-}
-
-var bench = New()
-var buf = make([]byte, 1024*1024)
-
-func benchmarkSize(b *testing.B, size int) {
-	b.SetBytes(int64(size))
-	sum := make([]byte, bench.Size())
-	for i := 0; i < b.N; i++ {
-		bench.Reset()
-		bench.Write(buf[:size])
-		bench.Sum(sum[:0])
-	}
-}
-
-func BenchmarkHash8Bytes(b *testing.B) {
-	benchmarkSize(b, 8)
-}
-
-func BenchmarkHash1K(b *testing.B) {
-	benchmarkSize(b, 1024)
-}
-
-func BenchmarkHash8K(b *testing.B) {
-	benchmarkSize(b, 8192)
-}
-
-func BenchmarkHash1M(b *testing.B) {
-	benchmarkSize(b, 1024*1024)
-}

pkg/crypto/sha1/sha1block.go

@@ -1,43 +0,0 @@
-// +build amd64,cgo
-// +build darwin windows
-
-/*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package sha1
-
-// #include <stdint.h>
-// #include <stdlib.h>
-// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
-import "C"
-import (
-	"unsafe"
-
-	"github.com/klauspost/cpuid"
-)
-
-func block(dig *digest, p []byte) {
-	switch true {
-	case cpuid.CPU.SSE3():
-		blockSSE3(dig, p)
-	default:
-		blockGeneric(dig, p)
-	}
-}
-
-func blockSSE3(dig *digest, p []byte) {
-	C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
-}

pkg/crypto/sha1/sha1block_generic.go

@@ -1,110 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file of
-// Golang project:
-//    https://github.com/golang/go/blob/master/LICENSE
-
-// Using this part of Minio codebase under the license
-// Apache License Version 2.0 with modifications
-
-package sha1
-
-const (
-	_K0 = 0x5A827999
-	_K1 = 0x6ED9EBA1
-	_K2 = 0x8F1BBCDC
-	_K3 = 0xCA62C1D6
-)
-
-// blockGeneric is a portable, pure Go version of the SHA1 block step.
-// It's used by sha1block_generic.go and tests.
-func blockGeneric(dig *digest, p []byte) {
-	var w [16]uint32
-
-	h0, h1, h2, h3, h4 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4]
-	for len(p) >= chunk {
-		// Can interlace the computation of w with the
-		// rounds below if needed for speed.
-		for i := 0; i < 16; i++ {
-			j := i * 4
-			w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3])
-		}
-
-		a, b, c, d, e := h0, h1, h2, h3, h4
-
-		// Each of the four 20-iteration rounds
-		// differs only in the computation of f and
-		// the choice of K (_K0, _K1, etc).
-		i := 0
-		for ; i < 16; i++ {
-			f := b&c | (^b)&d
-			a5 := a<<5 | a>>(32-5)
-			b30 := b<<30 | b>>(32-30)
-			t := a5 + f + e + w[i&0xf] + _K0
-			a, b, c, d, e = t, a, b30, c, d
-		}
-		for ; i < 20; i++ {
-			tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
-			w[i&0xf] = tmp<<1 | tmp>>(32-1)
-
-			f := b&c | (^b)&d
-			a5 := a<<5 | a>>(32-5)
-			b30 := b<<30 | b>>(32-30)
-			t := a5 + f + e + w[i&0xf] + _K0
-			a, b, c, d, e = t, a, b30, c, d
-		}
-		for ; i < 40; i++ {
-			tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
-			w[i&0xf] = tmp<<1 | tmp>>(32-1)
-			f := b ^ c ^ d
-			a5 := a<<5 | a>>(32-5)
-			b30 := b<<30 | b>>(32-30)
-			t := a5 + f + e + w[i&0xf] + _K1
-			a, b, c, d, e = t, a, b30, c, d
-		}
-		for ; i < 60; i++ {
-			tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
-			w[i&0xf] = tmp<<1 | tmp>>(32-1)
-			f := ((b | c) & d) | (b & c)
-
-			a5 := a<<5 | a>>(32-5)
-			b30 := b<<30 | b>>(32-30)
-			t := a5 + f + e + w[i&0xf] + _K2
-			a, b, c, d, e = t, a, b30, c, d
-		}
-		for ; i < 80; i++ {
-			tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
-			w[i&0xf] = tmp<<1 | tmp>>(32-1)
-			f := b ^ c ^ d
-			a5 := a<<5 | a>>(32-5)
-			b30 := b<<30 | b>>(32-30)
-			t := a5 + f + e + w[i&0xf] + _K3
-			a, b, c, d, e = t, a, b30, c, d
-		}
-
-		h0 += a
-		h1 += b
-		h2 += c
-		h3 += d
-		h4 += e
-
-		p = p[chunk:]
-	}
-	dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] = h0, h1, h2, h3, h4
-}

pkg/crypto/sha1/sha1block_linux.go

@@ -1,50 +0,0 @@
-// +build linux,amd64,cgo
-
-/*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package sha1
-
-// #cgo CFLAGS: -DHAS_AVX2
-// #include <stdint.h>
-// #include <stdlib.h>
-// void sha1_transform(int32_t *hash, const char* input, size_t num_blocks);
-// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
-import "C"
-import (
-	"unsafe"
-
-	"github.com/klauspost/cpuid"
-)
-
-func block(dig *digest, p []byte) {
-	switch true {
-	case cpuid.CPU.AVX2():
-		blockAVX2(dig, p)
-	case cpuid.CPU.SSE3():
-		blockSSE3(dig, p)
-	default:
-		blockGeneric(dig, p)
-	}
-}
-
-func blockAVX2(dig *digest, p []byte) {
-	C.sha1_transform((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
-}
-
-func blockSSE3(dig *digest, p []byte) {
-	C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
-}

pkg/crypto/sha256/sha256.go

@@ -1,4 +1,4 @@
-// +build darwin windows 386 arm !cgo
+// +build freebsd darwin windows 386 arm !cgo
 
 /*
  * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.

pkg/crypto/sha512/sha512.go

@@ -1,4 +1,4 @@
-// +build darwin windows 386 arm !cgo
+// +build freebsd darwin windows 386 arm !cgo
 
 /*
  * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.

pkg/disk/type_darwin.go

@@ -37,7 +37,7 @@ func getFSType(path string) (string, error) {
 	}
 	fsTypeHex := strconv.FormatUint(uint64(s.Type), 16)
 	fsTypeString, ok := fsType2StringMap[fsTypeHex]
-	if ok == false {
+	if !ok {
 		return "UNKNOWN", nil
 	}
 	return fsTypeString, nil

pkg/disk/type_unix.go

@@ -1,3 +1,5 @@
+// +build freebsd
+
 /*
  * Minio Cloud Storage, (C) 2015 Minio, Inc.
  *
@@ -14,33 +16,29 @@
  * limitations under the License.
  */
 
-package metadata
+package disk
 
 import (
-	"testing"
-
-	. "gopkg.in/check.v1"
+	"strconv"
+	"syscall"
 )
 
-func Test(t *testing.T) { TestingT(t) }
-
-type MySuite struct{}
-
-var _ = Suite(&MySuite{})
-
-func (s *MySuite) TestCache(c *C) {
-	cache := NewCache()
-	data := []byte("Hello, world!")
-	ok := cache.Set("filename", data)
-
-	c.Assert(ok, Equals, true)
-	storedata := cache.Get("filename")
-
-	c.Assert(ok, Equals, true)
-	c.Assert(data, DeepEquals, storedata)
-
-	cache.Delete("filename")
-
-	ok = cache.Exists("filename")
-	c.Assert(ok, Equals, false)
+// fsType2StringMap - list of filesystems supported by donut on linux
+var fsType2StringMap = map[string]string{
+	"35": "UFS",
+}
+
+// getFSType returns the filesystem type of the underlying mounted filesystem
+func getFSType(path string) (string, error) {
+	s := syscall.Statfs_t{}
+	err := syscall.Statfs(path, &s)
+	if err != nil {
+		return "", err
+	}
+	fsTypeHex := strconv.FormatInt(int64(s.Type), 16)
+	fsTypeString, ok := fsType2StringMap[fsTypeHex]
+	if ok == false {
+		return "UNKNOWN", nil
+	}
+	return fsTypeString, nil
 }

pkg/erasure/.gitignore

@@ -1 +0,0 @@
-*.syso

pkg/erasure/INSTALLGO.md

@@ -1,64 +0,0 @@
-## Ubuntu (Kylin) 14.04
-### Build Dependencies
-This installation document assumes Ubuntu 14.04+ on x86-64 platform.
-
-##### Install Git, GCC, yasm
-```sh
-$ sudo apt-get install git build-essential yasm
-```
-
-##### Install Go 1.6+
-
-Download Go 1.6+ from [https://golang.org/dl/](https://golang.org/dl/).
-
-```sh
-$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz
-$ mkdir -p ${HOME}/bin/
-$ mkdir -p ${HOME}/go/
-$ tar -C ${HOME}/bin/ -xzf go1.6.linux-amd64.tar.gz
-```
-##### Setup GOROOT and GOPATH
-
-Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
-and GOPATH specifies the location of your project workspace.
-
-```sh
-$ export GOROOT=${HOME}/bin/go
-$ export GOPATH=${HOME}/go
-$ export PATH=${HOME}/bin/go/bin:${GOPATH}/bin:$PATH
-```
-
-## OS X (Yosemite) 10.10
-### Build Dependencies
-This installation document assumes OS X Yosemite 10.10+ on x86-64 platform.
-
-##### Install brew
-```sh
-$ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-```
-
-##### Install Git, Python
-```sh
-$ brew install git python yasm
-```
-
-##### Install Go 1.6+
-
-Install golang binaries using `brew`
-
-```sh
-$ brew install go
-$ mkdir -p $HOME/go
-```
-
-##### Setup GOROOT and GOPATH
-
-Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
-and GOPATH specifies the location of your project workspace.
-
-```sh
-$ export GOPATH=${HOME}/go
-$ export GOVERSION=$(brew list go | head -n 1 | cut -d '/' -f 6)
-$ export GOROOT=$(brew --prefix)/Cellar/go/${GOVERSION}/libexec
-$ export PATH=${GOPATH}/bin:$PATH
-```

pkg/erasure/LICENSE.INTEL

@@ -1,26 +0,0 @@
-  Copyright(c) 2011-2014 Intel Corporation All rights reserved.
-
-  Redistribution and use in source and binary forms, with or without
-  modification, are permitted provided that the following conditions 
-  are met:
-    * Redistributions of source code must retain the above copyright
-      notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright
-      notice, this list of conditions and the following disclaimer in
-      the documentation and/or other materials provided with the
-      distribution.
-    * Neither the name of Intel Corporation nor the names of its
-      contributors may be used to endorse or promote products derived
-      from this software without specific prior written permission.
-
-  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pkg/erasure/LICENSE.MINIO

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

pkg/erasure/README.md

@@ -1,25 +0,0 @@
-## Introduction
-
-Erasure is an open source Golang library written on top of ISAL (Intel Intelligent Storage Library) released under [Apache license v2](./LICENSE)
-
-### Developers
-* [Get Source](./CONTRIBUTING.md)
-* [Build Dependencies](./BUILDDEPS.md)
-* [Development Workflow](./CONTRIBUTING.md#developer-guidelines)
-* [Developer discussions and bugs](https://github.com/minio/minio/issues)
-
-### Supported platforms
-
-| Name  | Supported |
-| ------------- | ------------- |
-| Linux  | Yes  |
-| Windows | Not yet |
-| Mac OSX | Yes |
-
-### Supported architectures
-
-| Arch | Supported |
-| ------------- | ------------- |
-| x86-64 | Yes |
-| arm64 | Not yet|
-| i386 | Never |

pkg/erasure/ctypes.go

@@ -1,62 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package erasure
-
-// #include <stdint.h>
-import "C"
-import (
-	"fmt"
-	"unsafe"
-)
-
-// intSlice2CIntArray converts Go int slice to C int array
-func intSlice2CIntArray(srcErrList []int) *C.int32_t {
-	if len(srcErrList) == 0 {
-		return (*C.int32_t)(unsafe.Pointer(nil))
-	}
-	var sizeErrInt = int(unsafe.Sizeof(srcErrList[0]))
-	switch sizeInt {
-	case sizeErrInt:
-		return (*C.int32_t)(unsafe.Pointer(&srcErrList[0]))
-	case sizeInt8:
-		int8Array := make([]int8, len(srcErrList))
-		for i, v := range srcErrList {
-			int8Array[i] = int8(v)
-		}
-		return (*C.int32_t)(unsafe.Pointer(&int8Array[0]))
-	case sizeInt16:
-		int16Array := make([]int16, len(srcErrList))
-		for i, v := range srcErrList {
-			int16Array[i] = int16(v)
-		}
-		return (*C.int32_t)(unsafe.Pointer(&int16Array[0]))
-	case sizeInt32:
-		int32Array := make([]int32, len(srcErrList))
-		for i, v := range srcErrList {
-			int32Array[i] = int32(v)
-		}
-		return (*C.int32_t)(unsafe.Pointer(&int32Array[0]))
-	case sizeInt64:
-		int64Array := make([]int64, len(srcErrList))
-		for i, v := range srcErrList {
-			int64Array[i] = int64(v)
-		}
-		return (*C.int32_t)(unsafe.Pointer(&int64Array[0]))
-	default:
-		panic(fmt.Sprintf("Unsupported: %d", sizeInt))
-	}
-}

pkg/erasure/doc.go

@@ -1,66 +0,0 @@
-// Package erasure is a Go wrapper for the Intel Intelligent Storage
-// Acceleration Library (Intel ISA-L).  Intel ISA-L is a CPU optimized
-// implementation of erasure coding algorithms.
-//
-// For more information on Intel ISA-L, please visit:
-// https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
-//
-// Usage:
-//
-// Encode encodes a block of data. The input is the original data. The output
-// is a 2 tuple containing (k + m) chunks of erasure encoded data and the
-// length of the original object.
-//
-// Decode decodes 2 tuple data containing (k + m) chunks back into its original form.
-// Additionally original block length should also be provided as input.
-//
-// Decoded data is exactly similar in length and content as the original data.
-//
-// Encoding data may be performed in 3 steps.
-//
-//  1. Create a parse set of encoder parameters
-//  2. Create a new encoder
-//  3. Encode data
-//
-// Decoding data is also performed in 3 steps.
-//
-//  1. Create a parse set of encoder parameters for validation
-//  2. Create a new encoder
-//  3. Decode data
-//
-// Erasure parameters contain three configurable elements:
-//  ValidateParams(k, m, technique int) (ErasureParams, error)
-//  k - Number of rows in matrix
-//  m - Number of colums in matrix
-//  technique - Matrix type, can be either Cauchy (recommended) or Vandermonde
-//  constraints: k + m < Galois Field (2^8)
-//
-// Choosing right parity and matrix technique is left for application to decide.
-//
-// But here are the few points to keep in mind
-//
-//  Matrix Type:
-//     - Vandermonde is most commonly used method for choosing coefficients in erasure
-//       encoding but does not guarantee invertable for every sub matrix.
-//     - Whereas Cauchy is our recommended method for choosing coefficients in erasure coding.
-//       Since any sub-matrix of a Cauchy matrix is invertable.
-//
-//  Total blocks:
-//     - Data blocks and Parity blocks should not be greater than 'Galois Field' (2^8)
-//
-// Example
-//
-// Creating and using an encoder
-//  var bytes []byte
-//  params := erasure.ValidateParams(10, 5)
-//  encoder := erasure.NewErasure(params)
-//  encodedData, length := encoder.Encode(bytes)
-//
-// Creating and using a decoder
-//  var encodedData [][]byte
-//  var length int
-//  params := erasure.ValidateParams(10, 5)
-//  encoder := erasure.NewErasure(params)
-//  originalData, err := encoder.Decode(encodedData, length)
-//
-package erasure

pkg/erasure/ec_minio_common.h

@@ -1,39 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __COMMON_H__
-#define __COMMON_H__
-
-#include <stdint.h>
-
-int32_t minio_init_encoder (int k, int m,
-                            unsigned char **encode_matrix,
-                            unsigned char **encode_tbls);
-
-int32_t minio_init_decoder (int32_t *error_index,
-                            int k, int n, int errs,
-                            unsigned char *encoding_matrix,
-                            unsigned char **decode_matrix,
-                            unsigned char **decode_tbls,
-                            uint32_t **decode_index);
-
-int32_t minio_get_source_target (int errs, int k, int m,
-                                 int32_t *error_index,
-                                 uint32_t *decode_index,
-                                 unsigned char **buffs,
-                                 unsigned char ***source,
-                                 unsigned char ***target);
-#endif /* __COMMON_H__ */

pkg/erasure/ec_minio_decode.c

@@ -1,142 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "isa-l.h"
-#include "ec_minio_common.h"
-
-static
-int32_t _minio_src_index_in_error (int r, int32_t *error_index, int errs)
-{
-        int i;
-        for (i = 0; i < errs; i++) {
-                if (error_index[i] == r) {
-                        // true
-                        return 1;
-                }
-        }
-        // false
-        return 0;
-}
-
-// Separate out source data and target buffers
-int32_t minio_get_source_target (int errs, int k, int m,
-                                 int32_t *error_index,
-                                 uint32_t *decode_index,
-                                 unsigned char **buffs,
-                                 unsigned char ***source,
-                                 unsigned char ***target)
-{
-        int i;
-        unsigned char *tmp_source[k];
-        unsigned char *tmp_target[m];
-
-        if (k < 0 || m < 0) {
-                return -1;
-        }
-
-        memset (tmp_source, 0, k);
-        memset (tmp_target, 0, m);
-
-        for (i = 0; i < k; i++) {
-                tmp_source[i] = (unsigned char *) buffs[decode_index[i]];
-        }
-
-        for (i = 0; i < m; i++) {
-                if (i < errs)
-                        tmp_target[i] = (unsigned char *) buffs[error_index[i]];
-        }
-
-        *source = tmp_source;
-        *target = tmp_target;
-
-	return 0;
-}
-
-/*
-  Generate decode matrix during the decoding phase
-*/
-
-int minio_init_decoder (int32_t *error_index,
-                        int k, int n, int errs,
-                        unsigned char *encode_matrix,
-                        unsigned char **decode_matrix,
-                        unsigned char **decode_tbls,
-                        uint32_t **decode_index)
-{
-        int i, j, r, l;
-
-        uint32_t      *tmp_decode_index = (uint32_t *) malloc(sizeof(uint32_t) * k);
-        unsigned char *input_matrix;
-        unsigned char *inverse_matrix;
-        unsigned char *tmp_decode_matrix;
-        unsigned char *tmp_decode_tbls;
-
-        input_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
-        inverse_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
-        tmp_decode_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);;
-        tmp_decode_tbls = (unsigned char *) malloc(sizeof(unsigned char) * k * n * 32);
-
-        for (i = 0, r = 0; i < k; i++, r++) {
-                while (_minio_src_index_in_error(r, error_index, errs))
-                        r++;
-                for (j = 0; j < k; j++) {
-                        input_matrix[k * i + j] = encode_matrix[k * r + j];
-                }
-                tmp_decode_index[i] = r;
-        }
-
-        // Not all vandermonde matrix can be inverted
-        if (gf_invert_matrix(input_matrix, inverse_matrix, k) < 0) {
-                free(tmp_decode_matrix);
-                free(tmp_decode_tbls);
-                free(tmp_decode_index);
-                return -1;
-        }
-
-        for (l = 0; l < errs; l++) {
-                if (error_index[l] < k) {
-                        // decoding matrix elements for data chunks
-                        for (j = 0; j < k; j++) {
-                                tmp_decode_matrix[k * l + j] =
-                                        inverse_matrix[k *
-                                                       error_index[l] + j];
-                        }
-                } else {
-                        // decoding matrix element for coding chunks
-                        for (i = 0; i < k; i++) {
-                                unsigned char s = 0;
-                                for (j = 0; j < k; j++) {
-                                        s ^= gf_mul(inverse_matrix[j * k + i],
-                                                    encode_matrix[k *
-                                                                  error_index[l] + j]);
-                                }
-                                tmp_decode_matrix[k * l + i] = s;
-                        }
-                }
-        }
-
-        ec_init_tables (k, errs, tmp_decode_matrix, tmp_decode_tbls);
-
-        *decode_matrix = tmp_decode_matrix;
-        *decode_tbls = tmp_decode_tbls;
-        *decode_index = tmp_decode_index;
-
-        return 0;
-}

pkg/erasure/ec_minio_encode.c

@@ -1,55 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-
-#include "isa-l.h"
-#include "ec_minio_common.h"
-
-/*
-  Generate encode matrix during the encoding phase
-*/
-
-int32_t minio_init_encoder (int k, int m, unsigned char **encode_matrix, unsigned char **encode_tbls)
-{
-        unsigned char *tmp_matrix;
-        unsigned char *tmp_tbls;
-
-        tmp_matrix = (unsigned char *) malloc (k * (k + m));
-        tmp_tbls = (unsigned char *) malloc (k * (k + m) * 32);
-
-        if (k < 5) {
-                /*
-                  Commonly used method for choosing coefficients in erasure
-                  encoding but does not guarantee invertable for every sub
-                  matrix.  For large k it is possible to find cases where the
-                  decode matrix chosen from sources and parity not in erasure
-                  are not invertable. Users may want to adjust for k > 5.
-                  -- Intel
-                */
-		gf_gen_rs_matrix (tmp_matrix, k + m, k);
-        } else {
-		gf_gen_cauchy1_matrix (tmp_matrix, k + m, k);
-        }
-
-	ec_init_tables(k, m, &tmp_matrix[k * k], tmp_tbls);
-
-        *encode_matrix = tmp_matrix;
-        *encode_tbls = tmp_tbls;
-
-        return 0;
-}

pkg/erasure/erasure_decode.go

@@ -1,123 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package erasure
-
-// #cgo CFLAGS: -O0
-// #include <stdlib.h>
-// #include "isa-l.h"
-// #include "ec_minio_common.h"
-import "C"
-import (
-	"errors"
-	"fmt"
-	"unsafe"
-)
-
-// Decode decodes erasure coded blocks of data into its original
-// form. Erasure coded data contains K data blocks and M parity
-// blocks. Decode can withstand data loss up to any M number of blocks.
-//
-// "encodedDataBlocks" is an array of K data blocks and M parity
-// blocks. Data blocks are position and order dependent. Missing blocks
-// are set to "nil". There must be at least "K" number of data|parity
-// blocks.
-//
-// "dataLen" is the length of original source data
-func (e *Erasure) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) {
-	e.mutex.Lock()
-	defer e.mutex.Unlock()
-
-	var source, target **C.uchar
-
-	k := int(e.params.K)
-	m := int(e.params.M)
-	n := k + m
-	// We need the data and parity blocks preserved in the same order. Missing blocks are set to nil.
-	if len(encodedDataBlocks) != n {
-		msg := fmt.Sprintf("Encoded data blocks slice must of length [%d]", n)
-		return nil, errors.New(msg)
-	}
-
-	// Length of a single encoded block
-	encodedBlockLen := GetEncodedBlockLen(dataLen, uint8(k))
-
-	// Keep track of errors per block.
-	missingEncodedBlocks := make([]int, n+1)
-	var missingEncodedBlocksCount int
-
-	// Check for the missing encoded blocks
-	for i := range encodedDataBlocks {
-		if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
-			missingEncodedBlocks[missingEncodedBlocksCount] = i
-			missingEncodedBlocksCount++
-		}
-	}
-
-	// Cannot reconstruct original data. Need at least M number of data or parity blocks.
-	if missingEncodedBlocksCount > m {
-		return nil, fmt.Errorf("Cannot reconstruct original data. Need at least [%d]  data or parity blocks", m)
-	}
-
-	// Convert from Go int slice to C int array
-	missingEncodedBlocksC := intSlice2CIntArray(missingEncodedBlocks[:missingEncodedBlocksCount])
-
-	// Allocate buffer for the missing blocks
-	for i := range encodedDataBlocks {
-		if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
-			encodedDataBlocks[i] = make([]byte, encodedBlockLen)
-		}
-	}
-
-	// If not already initialized, recompute and cache
-	if e.decodeMatrix == nil || e.decodeTbls == nil || e.decodeIndex == nil {
-		var decodeMatrix, decodeTbls *C.uchar
-		var decodeIndex *C.uint32_t
-
-		C.minio_init_decoder(missingEncodedBlocksC, C.int(k), C.int(n), C.int(missingEncodedBlocksCount),
-			e.encodeMatrix, &decodeMatrix, &decodeTbls, &decodeIndex)
-
-		// cache this for future needs
-		e.decodeMatrix = decodeMatrix
-		e.decodeTbls = decodeTbls
-		e.decodeIndex = decodeIndex
-	}
-
-	// Make a slice of pointers to encoded blocks. Necessary to bridge to the C world.
-	pointers := make([]*byte, n)
-	for i := range encodedDataBlocks {
-		pointers[i] = &encodedDataBlocks[i][0]
-	}
-
-	// Get pointers to source "data" and target "parity" blocks from the output byte array.
-	ret := C.minio_get_source_target(C.int(missingEncodedBlocksCount), C.int(k), C.int(m), missingEncodedBlocksC,
-		e.decodeIndex, (**C.uchar)(unsafe.Pointer(&pointers[0])), &source, &target)
-	if int(ret) == -1 {
-		return nil, errors.New("Unable to decode data")
-	}
-
-	// Decode data
-	C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(missingEncodedBlocksCount), e.decodeTbls,
-		source, target)
-
-	// Allocate buffer to output buffer
-	decodedData = make([]byte, 0, encodedBlockLen*int(k))
-	for i := 0; i < int(k); i++ {
-		decodedData = append(decodedData, encodedDataBlocks[i]...)
-	}
-
-	return decodedData[:dataLen], nil
-}

pkg/erasure/erasure_encode.go

@@ -1,174 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package erasure
-
-// #include <stdlib.h>
-// #include "isa-l.h"
-// #include "ec_minio_common.h"
-import "C"
-import (
-	"errors"
-	"sync"
-	"unsafe"
-)
-
-// Block alignment
-const (
-	SIMDAlign = 32
-)
-
-// Params is a configuration set for building an encoder. It is created using ValidateParams().
-type Params struct {
-	K uint8
-	M uint8
-}
-
-// Erasure is an object used to encode and decode data.
-type Erasure struct {
-	params                   *Params
-	encodeMatrix, encodeTbls *C.uchar
-	decodeMatrix, decodeTbls *C.uchar
-	decodeIndex              *C.uint32_t
-	mutex                    *sync.Mutex
-}
-
-// ValidateParams creates an Params object.
-//
-// k and m represent the matrix size, which corresponds to the protection level
-// technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde.
-//
-func ValidateParams(k, m uint8) (*Params, error) {
-	if k < 1 {
-		return nil, errors.New("k cannot be zero")
-	}
-
-	if m < 1 {
-		return nil, errors.New("m cannot be zero")
-	}
-
-	if k+m > 255 {
-		return nil, errors.New("(k + m) cannot be bigger than Galois field GF(2^8) - 1")
-	}
-
-	return &Params{
-		K: k,
-		M: m,
-	}, nil
-}
-
-// NewErasure creates an encoder object with a given set of parameters.
-func NewErasure(ep *Params) *Erasure {
-	var k = C.int(ep.K)
-	var m = C.int(ep.M)
-
-	var encodeMatrix *C.uchar
-	var encodeTbls *C.uchar
-
-	C.minio_init_encoder(k, m, &encodeMatrix, &encodeTbls)
-
-	return &Erasure{
-		params:       ep,
-		encodeMatrix: encodeMatrix,
-		encodeTbls:   encodeTbls,
-		decodeMatrix: nil,
-		decodeTbls:   nil,
-		decodeIndex:  nil,
-		mutex:        new(sync.Mutex),
-	}
-}
-
-// GetEncodedBlocksLen - total length of all encoded blocks
-func GetEncodedBlocksLen(inputLen int, k, m uint8) (outputLen int) {
-	outputLen = GetEncodedBlockLen(inputLen, k) * int(k+m)
-	return outputLen
-}
-
-// GetEncodedBlockLen - length per block of encoded blocks
-func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) {
-	alignment := int(k) * SIMDAlign
-	remainder := inputLen % alignment
-
-	paddedInputLen := inputLen
-	if remainder != 0 {
-		paddedInputLen = inputLen + (alignment - remainder)
-	}
-	encodedOutputLen = paddedInputLen / int(k)
-	return encodedOutputLen
-}
-
-// Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
-// Output is [k+m][]blocks of data and parity slices.
-func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
-	e.mutex.Lock()
-	defer e.mutex.Unlock()
-
-	k := int(e.params.K) // "k" data blocks
-	m := int(e.params.M) // "m" parity blocks
-	n := k + m           // "n" total encoded blocks
-
-	// Length of a single encoded chunk.
-	// Total number of encoded chunks = "k" data  + "m" parity blocks
-	encodedBlockLen := GetEncodedBlockLen(len(inputData), uint8(k))
-
-	// Length of total number of "k" data chunks
-	encodedDataBlocksLen := encodedBlockLen * k
-
-	// Length of extra padding required for the data blocks.
-	encodedDataBlocksPadLen := encodedDataBlocksLen - len(inputData)
-
-	// Extend inputData buffer to accommodate coded data blocks if necesssary
-	if encodedDataBlocksPadLen > 0 {
-		padding := make([]byte, encodedDataBlocksPadLen)
-		// Expand with new padded blocks to the byte array
-		inputData = append(inputData, padding...)
-	}
-
-	// Extend inputData buffer to accommodate coded parity blocks
-	{ // Local Scope
-		encodedParityBlocksLen := encodedBlockLen * m
-		parityBlocks := make([]byte, encodedParityBlocksLen)
-		inputData = append(inputData, parityBlocks...)
-	}
-
-	// Allocate memory to the "encoded blocks" return buffer
-	encodedBlocks = make([][]byte, n) // Return buffer
-
-	// Necessary to bridge Go to the C world. C requires 2D arry of pointers to
-	// byte array. "encodedBlocks" is a 2D slice.
-	pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.
-
-	// Copy data block slices to encoded block buffer
-	for i := 0; i < k; i++ {
-		encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
-		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
-	}
-
-	// Copy erasure block slices to encoded block buffer
-	for i := k; i < n; i++ {
-		encodedBlocks[i] = make([]byte, encodedBlockLen)
-		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
-	}
-
-	// Erasure code the data into K data blocks and M parity
-	// blocks. Only the parity blocks are filled. Data blocks remain
-	// intact.
-	C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
-		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
-		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks
-
-	return encodedBlocks, nil
-}

pkg/erasure/erasure_test.go

@@ -1,82 +0,0 @@
-// +build !windows
-
-/*
- * Minio Cloud Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package erasure
-
-import (
-	"bytes"
-	"testing"
-
-	. "gopkg.in/check.v1"
-)
-
-type MySuite struct{}
-
-var _ = Suite(&MySuite{})
-
-func Test(t *testing.T) { TestingT(t) }
-
-const (
-	k = 10
-	m = 5
-)
-
-func corruptChunks(chunks [][]byte, errorIndex []int) [][]byte {
-	for _, err := range errorIndex {
-		chunks[err] = nil
-	}
-	return chunks
-}
-
-func (s *MySuite) TestEncodeDecodeFailure(c *C) {
-	ep, err := ValidateParams(k, m)
-	c.Assert(err, IsNil)
-
-	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
-
-	e := NewErasure(ep)
-	chunks, err := e.Encode(data)
-	c.Assert(err, IsNil)
-
-	errorIndex := []int{0, 3, 5, 9, 11, 13}
-	chunks = corruptChunks(chunks, errorIndex)
-
-	_, err = e.Decode(chunks, len(data))
-	c.Assert(err, Not(IsNil))
-}
-
-func (s *MySuite) TestEncodeDecodeSuccess(c *C) {
-	ep, err := ValidateParams(k, m)
-	c.Assert(err, IsNil)
-
-	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
-
-	e := NewErasure(ep)
-	chunks, err := e.Encode(data)
-	c.Assert(err, IsNil)
-
-	errorIndex := []int{0, 3, 5, 9, 13}
-	chunks = corruptChunks(chunks, errorIndex)
-
-	recoveredData, err := e.Decode(chunks, len(data))
-	c.Assert(err, IsNil)
-
-	if !bytes.Equal(data, recoveredData) {
-		c.Fatalf("Recovered data mismatches with original data")
-	}
-}

pkg/erasure/stdint.go

@@ -1,38 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package erasure
-
-//
-// int sizeInt()
-// {
-//      return sizeof(int);
-// }
-import "C"
-import "unsafe"
-
-var (
-	// See http://golang.org/ref/spec#Numeric_types
-	sizeInt = int(C.sizeInt())
-	// SizeInt8  is the byte size of a int8.
-	sizeInt8 = int(unsafe.Sizeof(int8(0)))
-	// SizeInt16 is the byte size of a int16.
-	sizeInt16 = int(unsafe.Sizeof(int16(0)))
-	// SizeInt32 is the byte size of a int32.
-	sizeInt32 = int(unsafe.Sizeof(int32(0)))
-	// SizeInt64 is the byte size of a int64.
-	sizeInt64 = int(unsafe.Sizeof(int64(0)))
-)

pkg/fs/api_suite_windows_test.go

@@ -1,461 +0,0 @@
-// +build windows
-
-/*
- * Minimalist Object Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package fs
-
-import (
-	"bytes"
-	"crypto/md5"
-	"encoding/base64"
-	"encoding/hex"
-	"encoding/xml"
-	"math/rand"
-	"strconv"
-
-	"gopkg.in/check.v1"
-)
-
-// APITestSuite - collection of API tests
-func APITestSuite(c *check.C, create func() Filesystem) {
-	testMakeBucket(c, create)
-	testMultipleObjectCreation(c, create)
-	testPaging(c, create)
-	testObjectOverwriteWorks(c, create)
-	testNonExistantBucketOperations(c, create)
-	testBucketRecreateFails(c, create)
-	testPutObjectInSubdir(c, create)
-	testListBuckets(c, create)
-	testListBucketsOrder(c, create)
-	testListObjectsTestsForNonExistantBucket(c, create)
-	testNonExistantObjectInBucket(c, create)
-	testGetDirectoryReturnsObjectNotFound(c, create)
-	testDefaultContentType(c, create)
-	testMultipartObjectCreation(c, create)
-	testMultipartObjectAbort(c, create)
-}
-
-func testMakeBucket(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-}
-
-func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-	uploadID, err := fs.NewMultipartUpload("bucket", "key")
-	c.Assert(err, check.IsNil)
-
-	completedParts := CompleteMultipartUpload{}
-	completedParts.Part = make([]CompletePart, 0)
-	for i := 1; i <= 10; i++ {
-		randomPerm := rand.Perm(10)
-		randomString := ""
-		for _, num := range randomPerm {
-			randomString = randomString + strconv.Itoa(num)
-		}
-
-		hasher := md5.New()
-		hasher.Write([]byte(randomString))
-		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
-
-		var calculatedmd5sum string
-		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
-			bytes.NewBufferString(randomString), nil)
-		c.Assert(err, check.IsNil)
-		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
-		completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedmd5sum})
-	}
-	completedPartsBytes, e := xml.Marshal(completedParts)
-	c.Assert(e, check.IsNil)
-	objectMetadata, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil)
-	c.Assert(err, check.IsNil)
-	c.Assert(objectMetadata.MD5, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10")
-}
-
-func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-	uploadID, err := fs.NewMultipartUpload("bucket", "key")
-	c.Assert(err, check.IsNil)
-
-	parts := make(map[int]string)
-	for i := 1; i <= 10; i++ {
-		randomPerm := rand.Perm(10)
-		randomString := ""
-		for _, num := range randomPerm {
-			randomString = randomString + strconv.Itoa(num)
-		}
-
-		hasher := md5.New()
-		hasher.Write([]byte(randomString))
-		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
-
-		var calculatedmd5sum string
-		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
-			bytes.NewBufferString(randomString), nil)
-		c.Assert(err, check.IsNil)
-		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
-		parts[i] = calculatedmd5sum
-	}
-	err = fs.AbortMultipartUpload("bucket", "key", uploadID)
-	c.Assert(err, check.IsNil)
-}
-
-func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
-	objects := make(map[string][]byte)
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-	for i := 0; i < 10; i++ {
-		randomPerm := rand.Perm(10)
-		randomString := ""
-		for _, num := range randomPerm {
-			randomString = randomString + strconv.Itoa(num)
-		}
-
-		hasher := md5.New()
-		hasher.Write([]byte(randomString))
-		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
-
-		key := "obj" + strconv.Itoa(i)
-		objects[key] = []byte(randomString)
-		objectMetadata, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil)
-		c.Assert(err, check.IsNil)
-		c.Assert(objectMetadata.MD5, check.Equals, expectedmd5Sumhex)
-	}
-
-	for key, value := range objects {
-		var byteBuffer bytes.Buffer
-		_, err := fs.GetObject(&byteBuffer, "bucket", key, 0, 0)
-		c.Assert(err, check.IsNil)
-		c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
-
-		metadata, err := fs.GetObjectMetadata("bucket", key)
-		c.Assert(err, check.IsNil)
-		c.Assert(metadata.Size, check.Equals, int64(len(value)))
-	}
-}
-
-func testPaging(c *check.C, create func() Filesystem) {
-	fs := create()
-	fs.MakeBucket("bucket")
-	result, err := fs.ListObjects("bucket", "", "", "", 0)
-	c.Assert(err, check.IsNil)
-	c.Assert(len(result.Objects), check.Equals, 0)
-	c.Assert(result.IsTruncated, check.Equals, false)
-	// check before paging occurs
-	for i := 0; i < 5; i++ {
-		key := "obj" + strconv.Itoa(i)
-		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
-		c.Assert(err, check.IsNil)
-		result, err = fs.ListObjects("bucket", "", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, i+1)
-		c.Assert(result.IsTruncated, check.Equals, false)
-	}
-	// check after paging occurs pages work
-	for i := 6; i <= 10; i++ {
-		key := "obj" + strconv.Itoa(i)
-		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
-		c.Assert(err, check.IsNil)
-		result, err = fs.ListObjects("bucket", "", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, 5)
-		c.Assert(result.IsTruncated, check.Equals, true)
-	}
-	// check paging with prefix at end returns less objects
-	{
-		_, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
-		c.Assert(err, check.IsNil)
-		fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
-		c.Assert(err, check.IsNil)
-		result, err = fs.ListObjects("bucket", "new", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, 2)
-	}
-
-	// check ordering of pages
-	{
-		result, err = fs.ListObjects("bucket", "", "", "", 1000)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
-	}
-
-	// check delimited results with delimiter and prefix
-	{
-		_, err = fs.CreateObject("bucket", "this/is/delimited", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
-		c.Assert(err, check.IsNil)
-		_, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
-		c.Assert(err, check.IsNil)
-		result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, 1)
-		c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
-	}
-
-	// check delimited results with delimiter without prefix
-	{
-		result, err = fs.ListObjects("bucket", "", "", "/", 1000)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
-		c.Assert(result.Prefixes[0], check.Equals, "this/")
-	}
-
-	// check results with Marker
-	{
-		result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[1].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj1")
-	}
-	// check ordering of results with prefix
-	{
-		result, err = fs.ListObjects("bucket", "obj", "", "", 1000)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[1].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj10")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj2")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj3")
-	}
-	// check ordering of results with prefix and no paging
-	{
-		result, err = fs.ListObjects("bucket", "new", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-	}
-}
-
-func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
-	fs := create()
-	fs.MakeBucket("bucket")
-
-	hasher1 := md5.New()
-	hasher1.Write([]byte("one"))
-	md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
-	md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
-	objectMetadata, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.IsNil)
-	c.Assert(md5Sum1hex, check.Equals, objectMetadata.MD5)
-
-	hasher2 := md5.New()
-	hasher2.Write([]byte("three"))
-	md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil))
-	_, err = fs.CreateObject("bucket", "object", md5Sum2, int64(len("three")), bytes.NewBufferString("three"), nil)
-	c.Assert(err, check.IsNil)
-
-	var bytesBuffer bytes.Buffer
-	length, err := fs.GetObject(&bytesBuffer, "bucket", "object", 0, 0)
-	c.Assert(err, check.IsNil)
-	c.Assert(length, check.Equals, int64(len("three")))
-	c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three")
-}
-
-func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
-	fs := create()
-	_, err := fs.CreateObject("bucket", "object", "", int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.Not(check.IsNil))
-}
-
-func testBucketRecreateFails(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("string")
-	c.Assert(err, check.IsNil)
-	err = fs.MakeBucket("string")
-	c.Assert(err, check.Not(check.IsNil))
-}
-
-func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-
-	hasher := md5.New()
-	hasher.Write([]byte("hello world"))
-	md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-	md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
-	objectMetadata, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
-	c.Assert(err, check.IsNil)
-	c.Assert(objectMetadata.MD5, check.Equals, md5Sum1hex)
-
-	var bytesBuffer bytes.Buffer
-	length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0)
-	c.Assert(err, check.IsNil)
-	c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
-	c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length)
-}
-
-func testListBuckets(c *check.C, create func() Filesystem) {
-	fs := create()
-
-	// test empty list
-	buckets, err := fs.ListBuckets()
-	c.Assert(err, check.IsNil)
-	c.Assert(len(buckets), check.Equals, 0)
-
-	// add one and test exists
-	err = fs.MakeBucket("bucket1")
-	c.Assert(err, check.IsNil)
-
-	buckets, err = fs.ListBuckets()
-	c.Assert(len(buckets), check.Equals, 1)
-	c.Assert(err, check.IsNil)
-
-	// add two and test exists
-	err = fs.MakeBucket("bucket2")
-	c.Assert(err, check.IsNil)
-
-	buckets, err = fs.ListBuckets()
-	c.Assert(len(buckets), check.Equals, 2)
-	c.Assert(err, check.IsNil)
-
-	// add three and test exists + prefix
-	err = fs.MakeBucket("bucket22")
-
-	buckets, err = fs.ListBuckets()
-	c.Assert(len(buckets), check.Equals, 3)
-	c.Assert(err, check.IsNil)
-}
-
-func testListBucketsOrder(c *check.C, create func() Filesystem) {
-	// if implementation contains a map, order of map keys will vary.
-	// this ensures they return in the same order each time
-	for i := 0; i < 10; i++ {
-		fs := create()
-		// add one and test exists
-		err := fs.MakeBucket("bucket1")
-		c.Assert(err, check.IsNil)
-		err = fs.MakeBucket("bucket2")
-		c.Assert(err, check.IsNil)
-		buckets, err := fs.ListBuckets()
-		c.Assert(err, check.IsNil)
-		c.Assert(len(buckets), check.Equals, 2)
-		c.Assert(buckets[0].Name, check.Equals, "bucket1")
-		c.Assert(buckets[1].Name, check.Equals, "bucket2")
-	}
-}
-
-func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) {
-	fs := create()
-	result, err := fs.ListObjects("bucket", "", "", "", 1000)
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(result.IsTruncated, check.Equals, false)
-	c.Assert(len(result.Objects), check.Equals, 0)
-}
-
-func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-
-	var byteBuffer bytes.Buffer
-	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
-	c.Assert(length, check.Equals, int64(0))
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
-	switch err := err.ToGoError().(type) {
-	case ObjectNotFound:
-		{
-			c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
-		}
-	default:
-		{
-			c.Assert(err, check.Equals, "fails")
-		}
-	}
-}
-
-func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-
-	_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
-	c.Assert(err, check.IsNil)
-
-	var byteBuffer bytes.Buffer
-	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
-	c.Assert(length, check.Equals, int64(0))
-	switch err := err.ToGoError().(type) {
-	case ObjectNotFound:
-		c.Assert(err.Bucket, check.Equals, "bucket")
-		c.Assert(err.Object, check.Equals, "dir1")
-	default:
-		// force a failure with a line number
-		c.Assert(err, check.Equals, "ObjectNotFound")
-	}
-	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
-
-	var byteBuffer2 bytes.Buffer
-	length, err = fs.GetObject(&byteBuffer, "bucket", "dir1/", 0, 0)
-	c.Assert(length, check.Equals, int64(0))
-	switch err := err.ToGoError().(type) {
-	case ObjectNotFound:
-		c.Assert(err.Bucket, check.Equals, "bucket")
-		c.Assert(err.Object, check.Equals, "dir1/")
-	default:
-		// force a failure with a line number
-		c.Assert(err, check.Equals, "ObjectNotFound")
-	}
-	c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
-}
-
-func testDefaultContentType(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-
-	// test empty
-	_, err = fs.CreateObject("bucket", "one", "", int64(len("one")), bytes.NewBufferString("one"), nil)
-	metadata, err := fs.GetObjectMetadata("bucket", "one")
-	c.Assert(err, check.IsNil)
-	c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
-}
-
-func testContentMD5Set(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-
-	// test md5 invalid
-	badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA"
-	calculatedmd5sum, err := fs.CreateObject("bucket", "one", badmd5Sum, int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum)
-
-	goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA="
-	calculatedmd5sum, err = fs.CreateObject("bucket", "two", goodmd5sum, int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.IsNil)
-	c.Assert(calculatedmd5sum, check.Equals, goodmd5sum)
-}

pkg/fs/dir.go

@@ -1,245 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package fs
-
-import (
-	"io"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-)
-
-const (
-	// listObjectsLimit - maximum list objects limit
-	listObjectsLimit = 1000
-)
-
-// IsDirEmpty - returns whether given directory is empty or not
-func IsDirEmpty(dirname string) (status bool, err error) {
-	f, err := os.Open(dirname)
-	if err == nil {
-		defer f.Close()
-		if _, err = f.Readdirnames(1); err == io.EOF {
-			status = true
-			err = nil
-		}
-	}
-
-	return
-}
-
-// IsDirExist - returns whether given directory is exist or not
-func IsDirExist(dirname string) (status bool, err error) {
-	fi, err := os.Lstat(dirname)
-	if err == nil {
-		status = fi.IsDir()
-	}
-
-	return
-}
-
-// byName implements sort.Interface for sorting os.FileInfo list
-type byName []os.FileInfo
-
-func (f byName) Len() int      { return len(f) }
-func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
-func (f byName) Less(i, j int) bool {
-	n1 := f[i].Name()
-	if f[i].IsDir() {
-		n1 = n1 + string(os.PathSeparator)
-	}
-
-	n2 := f[j].Name()
-	if f[j].IsDir() {
-		n2 = n2 + string(os.PathSeparator)
-	}
-
-	return n1 < n2
-}
-
-// ObjectInfo - object info
-type ObjectInfo struct {
-	Name         string
-	ModifiedTime time.Time
-	Checksum     string
-	Size         int64
-	IsDir        bool
-	Err          error
-}
-
-// readDir - read 'scanDir' directory.  It returns list of ObjectInfo where
-//           each object name is appended with 'namePrefix'
-func readDir(scanDir, namePrefix string) (objInfos []ObjectInfo) {
-	f, err := os.Open(scanDir)
-	if err != nil {
-		objInfos = append(objInfos, ObjectInfo{Err: err})
-		return
-	}
-
-	fis, err := f.Readdir(-1)
-	if err != nil {
-		f.Close()
-		objInfos = append(objInfos, ObjectInfo{Err: err})
-		return
-	}
-
-	f.Close()
-	sort.Sort(byName(fis))
-
-	// make []ObjectInfo from []FileInfo
-	for _, fi := range fis {
-		name := fi.Name()
-		if namePrefix != "" {
-			name = namePrefix + "/" + name
-		}
-
-		if fi.IsDir() {
-			name += "/"
-		}
-
-		objInfos = append(objInfos, ObjectInfo{
-			Name:         name,
-			ModifiedTime: fi.ModTime(),
-			Checksum:     "",
-			Size:         fi.Size(),
-			IsDir:        fi.IsDir(),
-		})
-	}
-
-	return
-}
-
-// ObjectInfoChannel - object info channel
-type ObjectInfoChannel struct {
-	ch        <-chan ObjectInfo
-	objInfo   *ObjectInfo
-	closed    bool
-	timeoutCh <-chan struct{}
-	timedOut  bool
-}
-
-func (oic *ObjectInfoChannel) Read() (ObjectInfo, bool) {
-	if oic.closed {
-		return ObjectInfo{}, false
-	}
-
-	if oic.objInfo == nil {
-		// first read
-		if oi, ok := <-oic.ch; ok {
-			oic.objInfo = &oi
-		} else {
-			oic.closed = true
-			return ObjectInfo{}, false
-		}
-	}
-
-	retObjInfo := *oic.objInfo
-	status := true
-	oic.objInfo = nil
-
-	// read once more to know whether it was last read
-	if oi, ok := <-oic.ch; ok {
-		oic.objInfo = &oi
-	} else {
-		oic.closed = true
-	}
-
-	return retObjInfo, status
-}
-
-// IsClosed - return whether channel is closed or not
-func (oic ObjectInfoChannel) IsClosed() bool {
-	if oic.objInfo != nil {
-		return false
-	}
-
-	return oic.closed
-}
-
-// IsTimedOut - return whether channel is closed due to timeout
-func (oic ObjectInfoChannel) IsTimedOut() bool {
-	if oic.timedOut {
-		return true
-	}
-
-	select {
-	case _, ok := <-oic.timeoutCh:
-		if ok {
-			oic.timedOut = true
-			return true
-		}
-		return false
-	default:
-		return false
-	}
-}
-
-// treeWalk - walk into 'scanDir' recursively when 'recursive' is true.
-//            It uses 'bucketDir' to get name prefix for object name.
-func treeWalk(scanDir, bucketDir string, recursive bool) ObjectInfoChannel {
-	objectInfoCh := make(chan ObjectInfo, listObjectsLimit)
-	timeoutCh := make(chan struct{}, 1)
-
-	// goroutine - retrieves directory entries, makes ObjectInfo and sends into the channel
-	go func() {
-		defer close(objectInfoCh)
-		defer close(timeoutCh)
-
-		// send function - returns true if ObjectInfo is sent
-		//                 within (time.Second * 15) else false on time-out
-		send := func(oi ObjectInfo) bool {
-			timer := time.After(time.Second * 15)
-			select {
-			case objectInfoCh <- oi:
-				return true
-			case <-timer:
-				timeoutCh <- struct{}{}
-				return false
-			}
-		}
-
-		namePrefix := strings.Replace(filepath.ToSlash(scanDir), filepath.ToSlash(bucketDir), "", 1)
-		if strings.HasPrefix(namePrefix, "/") {
-			/* remove beginning "/" */
-			namePrefix = namePrefix[1:]
-		}
-
-		for objInfos := readDir(scanDir, namePrefix); len(objInfos) > 0; {
-			var objInfo ObjectInfo
-			objInfo, objInfos = objInfos[0], objInfos[1:]
-			if !send(objInfo) {
-				return
-			}
-
-			if objInfo.IsDir && recursive {
-				scanDir := filepath.Join(bucketDir, filepath.FromSlash(objInfo.Name))
-
-				namePrefix = strings.Replace(filepath.ToSlash(scanDir), filepath.ToSlash(bucketDir), "", 1)
-				if strings.HasPrefix(namePrefix, "/") {
-					/* remove beginning "/" */
-					namePrefix = namePrefix[1:]
-				}
-
-				objInfos = append(readDir(scanDir, namePrefix), objInfos...)
-			}
-		}
-	}()
-
-	return ObjectInfoChannel{ch: objectInfoCh, timeoutCh: timeoutCh}
-}

pkg/fs/fs-backend-metadata.go

@@ -1,57 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package fs
-
-import (
-	"github.com/minio/minio/pkg/probe"
-	"github.com/minio/minio/pkg/quick"
-)
-
-var multipartsMetadataPath string
-
-// SetFSMultipartsMetadataPath - set custom multiparts session
-// metadata path.
-func setFSMultipartsMetadataPath(metadataPath string) {
-	multipartsMetadataPath = metadataPath
-}
-
-// saveMultipartsSession - save multiparts
-func saveMultipartsSession(multiparts Multiparts) *probe.Error {
-	qc, err := quick.New(multiparts)
-	if err != nil {
-		return err.Trace()
-	}
-	if err := qc.Save(multipartsMetadataPath); err != nil {
-		return err.Trace()
-	}
-	return nil
-}
-
-// loadMultipartsSession load multipart session file
-func loadMultipartsSession() (*Multiparts, *probe.Error) {
-	multiparts := &Multiparts{}
-	multiparts.Version = "1"
-	multiparts.ActiveSession = make(map[string]*MultipartSession)
-	qc, err := quick.New(multiparts)
-	if err != nil {
-		return nil, err.Trace()
-	}
-	if err := qc.Load(multipartsMetadataPath); err != nil {
-		return nil, err.Trace()
-	}
-	return qc.Data().(*Multiparts), nil
-}