Use normal color instead of forced white for users who have bright terminal background (#2200)

Fixes #2201
Fixes #2204

.travis.yml

@@ -4,9 +4,10 @@ language: go
 
 os:
 - linux
-- osx
+# Enable OSX builds when travis service is back.
+#- osx
 
-osx_image: xcode7.2
+#osx_image: xcode7.2
 
 env:
 - ARCH=x86_64

INSTALLGO.md

@@ -53,7 +53,7 @@ $ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/maste
 $ brew install git python
 ```
 
-##### Install Go 1.5+
+##### Install Go 1.6+
 
 Install golang binaries using `brew`
 

Makefile

@@ -1,8 +1,6 @@
 LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
-DOCKER_BIN := $(shell which docker)
 PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
-DOCKER_LDFLAGS := '$(LDFLAGS) -extldflags "-static"'
 BUILD_LDFLAGS := '$(LDFLAGS)'
 TAG := latest
 
@@ -62,10 +60,6 @@ checks:
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
-checkdocker:
-	@echo "Checking if docker is installed.. "
-	@if [ -z ${DOCKER_BIN} ]; then echo "Docker not installed, cannot build docker image. Please install 'sudo apt-get install docker.io'" && exit 1; else echo "Docker installed at ${DOCKER_BIN}."; fi;
-
 getdeps: checks
 	@go get -u github.com/golang/lint/golint && echo "Installed golint:"
 	@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
@@ -133,13 +127,6 @@ pkg-list:
 
 install: gomake-all
 
-dockerimage: checkdocker getdeps verifiers $(UI_ASSETS)
-	@echo "Building docker image:" minio:$(TAG)
-	@GO15VENDOREXPERIMENT=1 GOOS=linux GOARCH=amd64 go build --ldflags $(DOCKER_LDFLAGS) -o docker/minio.dockerimage
-	@cd docker; mkdir -p export; sudo docker build --rm --tag=minio/minio:$(TAG) .
-	@rmdir docker/export
-	@rm docker/minio.dockerimage
-
 release: verifiers
 	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
 

README.md

@@ -98,20 +98,22 @@ Start minio server.
 ~~~
 $ minio server ~/Photos
 
-AccessKey: WLGDGYAQYIGI833EV05A  SecretKey: BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF  Region: us-east-1
+Endpoint:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
+AccessKey: USWUXHGYZQYFYFFIT3RE
+SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
+Region:    us-east-1
 
-Minio Object Storage:
-    http://127.0.0.1:9000
-    http://10.1.10.177:9000
+Browser Access:
+   http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 
-Minio Browser:
-    http://127.0.0.1:9000
-    http://10.1.10.177:9000
+Command-line Access: https://docs.minio.io/docs/minio-client-quick-start-guide
+   $ ./mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
 
-To configure Minio Client:
-    $ wget https://dl.minio.io/client/mc/release/darwin-amd64/mc
-    $ chmod 755 mc
-    $ ./mc config host add myminio http://localhost:9000 WLGDGYAQYIGI833EV05A BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF
+Object API (Amazon S3 compatible):
+   Go:         https://docs.minio.io/docs/golang-client-quickstart-guide
+   Java:       https://docs.minio.io/docs/java-client-quickstart-guide
+   Python:     https://docs.minio.io/docs/python-client-quickstart-guide
+   JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide
 ~~~
 
 #### How to use AWS CLI with Minio?
@@ -206,7 +208,7 @@ Bucket 's3://mybucket/' created
 To copy an object to bucket.
 ```
 $ s3cmd put newfile s3://testbucket
-upload: 'newfile' -> 's3://testbucket/newfile'  
+upload: 'newfile' -> 's3://testbucket/newfile'
 ```
 
 To copy an object to local system.

access-key.go

@@ -19,7 +19,6 @@ package main
 import (
 	"crypto/rand"
 	"encoding/base64"
-	"fmt"
 	"regexp"
 )
 
@@ -29,13 +28,6 @@ type credential struct {
 	SecretAccessKey string `json:"secretKey"`
 }
 
-// stringer colorized access keys.
-func (a credential) String() string {
-	accessStr := colorMagenta("AccessKey: ") + colorWhite(a.AccessKeyID)
-	secretStr := colorMagenta("SecretKey: ") + colorWhite(a.SecretAccessKey)
-	return fmt.Sprint(accessStr + "  " + secretStr)
-}
-
 const (
 	minioAccessID = 20
 	minioSecretID = 40

api-errors.go

@@ -74,6 +74,7 @@ const (
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNotImplemented
+	ErrPreconditionFailed
 	ErrRequestTimeTooSkewed
 	ErrSignatureDoesNotMatch
 	ErrMethodNotAllowed
@@ -104,11 +105,20 @@ const (
 	ErrBucketAlreadyOwnedByYou
 	// Add new error codes here.
 
+	// S3 extended errors.
+	ErrContentSHA256Mismatch
+	// Add new extended error codes here.
+
 	// Minio extended errors.
 	ErrReadQuorum
 	ErrWriteQuorum
 	ErrStorageFull
 	ErrObjectExistsAsDirectory
+	ErrPolicyNesting
+	ErrInvalidObjectName
+	// Add new extended error codes here.
+	// Please open a https://github.com/minio/minio/issues before adding
+	// new error codes here.
 )
 
 // error code to APIError structure, these fields carry respective
@@ -131,17 +141,17 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInvalidMaxUploads: {
 		Code:           "InvalidArgument",
-		Description:    "Argument maxUploads must be an integer between 0 and 2147483647.",
+		Description:    "Argument max-uploads must be an integer between 0 and 2147483647",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidMaxKeys: {
 		Code:           "InvalidArgument",
-		Description:    "Argument maxKeys must be an integer between 0 and 2147483647.",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidMaxParts: {
 		Code:           "InvalidArgument",
-		Description:    "Argument maxParts must be an integer between 1 and 10000.",
+		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartNumberMarker: {
@@ -206,7 +216,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInvalidRange: {
 		Code:           "InvalidRange",
-		Description:    "The requested range cannot be satisfied.",
+		Description:    "The requested range is not satisfiable",
 		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
 	},
 	ErrMalformedXML: {
@@ -231,7 +241,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrNoSuchBucket: {
 		Code:           "NoSuchBucket",
-		Description:    "The specified bucket does not exist.",
+		Description:    "The specified bucket does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrNoSuchBucketPolicy: {
@@ -251,9 +261,14 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrNotImplemented: {
 		Code:           "NotImplemented",
-		Description:    "A header you provided implies functionality that is not implemented.",
+		Description:    "A header you provided implies functionality that is not implemented",
 		HTTPStatusCode: http.StatusNotImplemented,
 	},
+	ErrPreconditionFailed: {
+		Code:           "PreconditionFailed",
+		Description:    "At least one of the pre-conditions you specified did not hold",
+		HTTPStatusCode: http.StatusPreconditionFailed,
+	},
 	ErrRequestTimeTooSkewed: {
 		Code:           "RequestTimeTooSkewed",
 		Description:    "The difference between the request time and the server's time is too large.",
@@ -261,7 +276,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrSignatureDoesNotMatch: {
 		Code:           "SignatureDoesNotMatch",
-		Description:    "The request signature we calculated does not match the signature you provided.",
+		Description:    "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrMethodNotAllowed: {
@@ -394,6 +409,14 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Your previous request to create the named bucket succeeded and you already own it.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+
+	/// S3 extensions.
+	ErrContentSHA256Mismatch: {
+		Code:           "XAmzContentSHA256Mismatch",
+		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
 	/// Minio extensions.
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
@@ -415,6 +438,16 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Multiple disks failures, unable to write data.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrPolicyNesting: {
+		Code:           "XMinioPolicyNesting",
+		Description:    "Policy nesting conflict has occurred.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrInvalidObjectName: {
+		Code:           "XMinioInvalidObjectName",
+		Description:    "Object name contains unsupported characters. Unsupported characters are `^*|\\\"",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -426,8 +459,15 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 	// Verify if the underlying error is signature mismatch.
-	if err == errSignatureMismatch {
-		return ErrSignatureDoesNotMatch
+	switch err {
+	case errSignatureMismatch:
+		apiErr = ErrSignatureDoesNotMatch
+	case errContentSHA256Mismatch:
+		apiErr = ErrContentSHA256Mismatch
+	}
+	if apiErr != ErrNone {
+		// If there was a match in the above switch case.
+		return apiErr
 	}
 	switch err.(type) {
 	case StorageFull:
@@ -449,7 +489,7 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 	case ObjectNotFound:
 		apiErr = ErrNoSuchKey
 	case ObjectNameInvalid:
-		apiErr = ErrNoSuchKey
+		apiErr = ErrInvalidObjectName
 	case InvalidUploadID:
 		apiErr = ErrNoSuchUpload
 	case InvalidPart:

api-headers.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"runtime"
 	"strconv"
+	"strings"
 )
 
 //// helpers
@@ -72,14 +73,17 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 	}
 
 	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
-
-	// for providing ranged content
-	if contentRange != nil {
-		if contentRange.start > 0 || contentRange.length > 0 {
-			// override content-length
-			w.Header().Set("Content-Length", strconv.FormatInt(contentRange.length, 10))
-			w.Header().Set("Content-Range", contentRange.String())
-			w.WriteHeader(http.StatusPartialContent)
+	for k, v := range objInfo.UserDefined {
+		if strings.HasPrefix(k, "X-Amz-Meta-") {
+			w.Header().Set(k, v)
 		}
 	}
+
+	// for providing ranged content
+	if contentRange != nil && contentRange.offsetBegin > -1 {
+		// Override content-length
+		w.Header().Set("Content-Length", strconv.FormatInt(contentRange.getLength(), 10))
+		w.Header().Set("Content-Range", contentRange.String())
+		w.WriteHeader(http.StatusPartialContent)
+	}
 }

api-resources.go

@@ -50,23 +50,6 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 	return
 }
 
-// Parse bucket url queries
-func getBucketResources(values url.Values) (listType int, prefix, marker, delimiter string, maxkeys int, encodingType string) {
-	if values.Get("list-type") != "" {
-		listType, _ = strconv.Atoi(values.Get("list-type"))
-	}
-	prefix = values.Get("prefix")
-	marker = values.Get("marker")
-	delimiter = values.Get("delimiter")
-	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
-	} else {
-		maxkeys = maxObjectList
-	}
-	encodingType = values.Get("encoding-type")
-	return
-}
-
 // Parse bucket url queries for ?uploads
 func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
 	prefix = values.Get("prefix")

api-response-multipart.go

@@ -0,0 +1,54 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// File carries any specific responses constructed/necessary in
+// multipart operations.
+package main
+
+import "net/http"
+
+// writeErrorResponsePartTooSmall - function is used specifically to
+// construct a proper error response during CompleteMultipartUpload
+// when one of the parts is < 5MB.
+// The requirement comes due to the fact that generic ErrorResponse
+// XML doesn't carry the additional fields required to send this
+// error. So we construct a new type which lies well within the scope
+// of this function.
+func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
+	// Represents additional fields necessary for ErrPartTooSmall S3 error.
+	type completeMultipartAPIError struct {
+		// Proposed size represents uploaded size of the part.
+		ProposedSize int64
+		// Minimum size allowed epresents the minimum size allowed per
+		// part. Defaults to 5MB.
+		MinSizeAllowed int64
+		// Part number of the part which is incorrect.
+		PartNumber int
+		// ETag of the part which is incorrect.
+		PartETag string
+		// Other default XML error responses.
+		APIErrorResponse
+	}
+	// Generate complete multipart error response.
+	errorResponse := getAPIErrorResponse(getAPIError(toAPIErrorCode(err)), r.URL.Path)
+	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
+	encodedErrorResponse := encodeResponse(cmpErrResp)
+	// Write error body
+	w.Write(encodedErrorResponse)
+	w.(http.Flusher).Flush()
+}
+
+// Add any other multipart specific responses here.

api-response.go

@@ -19,6 +19,7 @@ package main
 import (
 	"encoding/xml"
 	"net/http"
+	"path"
 	"time"
 )
 
@@ -255,7 +256,7 @@ type DeleteObjectsResponse struct {
 
 // getLocation get URL location.
 func getLocation(r *http.Request) string {
-	return r.URL.Path
+	return path.Clean(r.URL.Path) // Clean any trailing slashes.
 }
 
 // getObjectLocation gets the relative URL for an object
@@ -497,13 +498,18 @@ func writeSuccessNoContent(w http.ResponseWriter) {
 // writeErrorRespone write error headers
 func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorCode APIErrorCode, resource string) {
 	error := getAPIError(errorCode)
-	// generate error response
-	errorResponse := getAPIErrorResponse(error, resource)
-	encodedErrorResponse := encodeResponse(errorResponse)
 	// set common headers
 	setCommonHeaders(w)
 	// write Header
 	w.WriteHeader(error.HTTPStatusCode)
+	writeErrorResponseNoHeader(w, req, errorCode, resource)
+}
+
+func writeErrorResponseNoHeader(w http.ResponseWriter, req *http.Request, errorCode APIErrorCode, resource string) {
+	error := getAPIError(errorCode)
+	// Generate error response.
+	errorResponse := getAPIErrorResponse(error, resource)
+	encodedErrorResponse := encodeResponse(errorResponse)
 	// HEAD should have no body, do not attempt to write to it
 	if req.Method != "HEAD" {
 		// write error body

api-router.go

@@ -48,7 +48,7 @@ func registerAPIRouter(mux *router.Router, api objectAPIHandlers) {
 	// GetObject
 	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
 	// CopyObject
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/).*?").HandlerFunc(api.CopyObjectHandler)
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
 	// PutObject
 	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
 	// DeleteObject

appveyor.yml

@@ -1,6 +1,10 @@
-# Operating system (build VM template)
-os: Visual Studio 2015
+# version format
+version: "{build}"
 
+# Operating system (build VM template)
+os: Windows Server 2012 R2
+
+# Platform.
 platform: x64
 
 clone_folder: c:\gopath\src\github.com\minio\minio
@@ -8,19 +12,13 @@ clone_folder: c:\gopath\src\github.com\minio\minio
 # environment variables
 environment:
   GOPATH: c:\gopath
-  GO_EXTLINK_ENABLED: 0
   GO15VENDOREXPERIMENT: 1
 
 # scripts that run after cloning repository
 install:
-  - '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64'
   - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
-  - rd C:\Go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.windows-amd64.zip
-  - 7z x go1.6.windows-amd64.zip -oC:\ >nul
   - go version
   - go env
-  - cd %GOPATH%\src\github.com\minio\minio
 
 # to run your custom scripts instead of automatic MSBuild
 build_script:

auth-handler.go

@@ -27,6 +27,11 @@ import (
 	"strings"
 )
 
+// Verify if the request http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD"
+func isRequestUnsignedPayload(r *http.Request) bool {
+	return r.Header.Get("x-amz-content-sha256") == unsignedPayload
+}
+
 // Verify if request has JWT.
 func isRequestJWT(r *http.Request) bool {
 	if _, ok := r.Header["Authorization"]; ok {
@@ -126,10 +131,18 @@ func isReqAuthenticated(r *http.Request) (s3Error APIErrorCode) {
 	// Populate back the payload.
 	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	validateRegion := true // Validate region.
+	var sha256sum string
+	// Skips calculating sha256 on the payload on server,
+	// if client requested for it.
+	if skipContentSha256Cksum(r) {
+		sha256sum = unsignedPayload
+	} else {
+		sha256sum = hex.EncodeToString(sum256(payload))
+	}
 	if isRequestSignatureV4(r) {
-		return doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		return doesSignatureMatch(sha256sum, r, validateRegion)
 	} else if isRequestPresignedSignatureV4(r) {
-		return doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		return doesPresignedSignatureMatch(sha256sum, r, validateRegion)
 	}
 	return ErrAccessDenied
 }

benchmark-utils_test.go

@@ -0,0 +1,384 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"io/ioutil"
+	"math"
+	"math/rand"
+	"strconv"
+	"testing"
+	"time"
+)
+
+// Benchmark utility functions for ObjectLayer.PutObject().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
+func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// PutObject returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum string
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for PutObject.
+	hasher := md5.New()
+	hasher.Write([]byte(textData))
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// insert the object.
+		md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if md5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// Benchmark utility functions for ObjectLayer.PutObjectPart().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObjectPart benchmark.
+func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	objSize := 128 * 1024 * 1024
+
+	// PutObjectPart returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum, uploadID string
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for NewMultipartUpload.
+	hasher := md5.New()
+	hasher.Write([]byte(textData))
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+	uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	var textPartData []byte
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObjectPart starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// insert the object.
+		totalPartsNR := int(math.Ceil(float64(objSize) / float64(partSize)))
+		for j := 0; j < totalPartsNR; j++ {
+			hasher.Reset()
+			if j < totalPartsNR-1 {
+				textPartData = textData[j*partSize : (j+1)*partSize-1]
+			} else {
+				textPartData = textData[j*partSize:]
+			}
+			hasher.Write([]byte(textPartData))
+			metadata := make(map[string]string)
+			metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+			md5Sum, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"])
+			if err != nil {
+				b.Fatal(err)
+			}
+			if md5Sum != metadata["md5Sum"] {
+				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+			}
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
+func benchmarkPutObjectPart(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// calling runPutObjectBenchmark which uses *testing.B and the object Layer to run the benchmark.
+	runBenchMark(b, objLayer)
+}
+
+// closure for returning the put object benchmark executor for given object size in bytes.
+func returnPutObjectPartBenchmark(objSize int) func(*testing.B, ObjectLayer) {
+	// FIXME: Avoid closure.
+	return func(b *testing.B, obj ObjectLayer) {
+		runPutObjectPartBenchmark(b, obj, objSize)
+	}
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
+func benchmarkPutObject(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// calling runPutObjectBenchmark which uses *testing.B and the object Layer to run the benchmark.
+	runBenchMark(b, objLayer)
+}
+
+// closure for returning the put object benchmark executor for given object size in bytes.
+func returnPutObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) {
+	// FIXME: Avoid closure.
+	return func(b *testing.B, obj ObjectLayer) {
+		runPutObjectBenchmark(b, obj, objSize)
+	}
+}
+
+// Benchmark utility functions for ObjectLayer.GetObject().
+// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
+func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// PutObject returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum string
+	for i := 0; i < 10; i++ {
+		// get text data generated for number of bytes equal to object size.
+		textData := generateBytesData(objSize)
+		// generate md5sum for the generated data.
+		// md5sum of the data to written is required as input for PutObject.
+		// PutObject is the functions which writes the data onto the FS/XL backend.
+		hasher := md5.New()
+		hasher.Write([]byte(textData))
+		metadata := make(map[string]string)
+		metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+		// insert the object.
+		md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if md5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+		}
+	}
+
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for GetObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		var buffer = new(bytes.Buffer)
+		err = obj.GetObject(bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer)
+		if err != nil {
+			b.Error(err)
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+
+}
+
+// randomly picks a character and returns its equivalent byte array.
+func getRandomByte() []byte {
+	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	// seeding the random number generator.
+	rand.Seed(time.Now().UnixNano())
+	var b byte
+	// pick a character randomly.
+	b = letterBytes[rand.Intn(len(letterBytes))]
+	return []byte{b}
+}
+
+// picks a random byte and repeats it to size bytes.
+func generateBytesData(size int) []byte {
+	// repeat the random character chosen size
+	return bytes.Repeat(getRandomByte(), size)
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
+func benchmarkGetObject(b *testing.B, instanceType string, runBenchMark func(b *testing.B, obj ObjectLayer)) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created.
+	defer removeRoots(disks)
+	// calling runGetObjectBenchmark which uses *testing.B and the object Layer to run the benchmark.
+	runBenchMark(b, objLayer)
+}
+
+// closure for returning the get object benchmark executor for given object size in bytes.
+// FIXME: Avoid closure.
+func returnGetObjectBenchmark(objSize int) func(*testing.B, ObjectLayer) {
+	return func(b *testing.B, obj ObjectLayer) {
+		runGetObjectBenchmark(b, obj, objSize)
+	}
+}
+
+// Parallel benchmark utility functions for ObjectLayer.PutObject().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
+func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// PutObject returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum string
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for PutObject.
+	hasher := md5.New()
+	hasher.Write([]byte(textData))
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		i := 0
+		for pb.Next() {
+			// insert the object.
+			md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
+			if err != nil {
+				b.Fatal(err)
+			}
+			if md5Sum != metadata["md5Sum"] {
+				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", md5Sum, metadata["md5Sum"])
+			}
+			i++
+		}
+	})
+
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// closure for returning the put object benchmark executor for given object size in bytes.
+func returnPutObjectBenchmarkParallel(objSize int) func(*testing.B, ObjectLayer) {
+	// FIXME: Avoid closure.
+	return func(b *testing.B, obj ObjectLayer) {
+		runPutObjectBenchmarkParallel(b, obj, objSize)
+	}
+}
+
+// Parallel benchmark utility functions for ObjectLayer.GetObject().
+// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
+func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// PutObject returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum string
+	for i := 0; i < 10; i++ {
+		// get text data generated for number of bytes equal to object size.
+		textData := generateBytesData(objSize)
+		// generate md5sum for the generated data.
+		// md5sum of the data to written is required as input for PutObject.
+		// PutObject is the functions which writes the data onto the FS/XL backend.
+		hasher := md5.New()
+		hasher.Write([]byte(textData))
+		metadata := make(map[string]string)
+		metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+		// insert the object.
+		md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if md5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+		}
+	}
+
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for GetObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		i := 0
+		for pb.Next() {
+			err = obj.GetObject(bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard)
+			if err != nil {
+				b.Error(err)
+			}
+			i++
+			if i == 10 {
+				i = 0
+			}
+		}
+	})
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+
+}
+
+// closure for returning the get object benchmark executor for given object size in bytes.
+// FIXME: Avoid closure.
+func returnGetObjectBenchmarkParallel(objSize int) func(*testing.B, ObjectLayer) {
+	return func(b *testing.B, obj ObjectLayer) {
+		runGetObjectBenchmarkParallel(b, obj, objSize)
+	}
+}

bucket-policy-handlers.go

@@ -21,14 +21,13 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"regexp"
 	"strings"
 
 	mux "github.com/gorilla/mux"
 )
 
 // maximum supported access policy size.
-const maxAccessPolicySize = 20 * 1024 * 1024 // 20KiB.
+const maxAccessPolicySize = 20 * 1024 // 20KiB.
 
 // Verify if a given action is valid for the url path based on the
 // existing bucket access policy.
@@ -65,41 +64,48 @@ func bucketPolicyMatchStatement(action string, resource string, conditions map[s
 // Verify if given action matches with policy statement.
 func bucketPolicyActionMatch(action string, statement policyStatement) bool {
 	for _, policyAction := range statement.Actions {
-		// Policy action can be a regex, validate the action with matching string.
-		matched, err := regexp.MatchString(policyAction, action)
-		fatalIf(err, "Invalid action \"%s\" in bucket policy.", action)
-		if matched {
+		if matched := actionMatch(policyAction, action); matched {
 			return true
 		}
 	}
 	return false
 }
 
-// Match function matches wild cards in 'pattern' for resource.
-func resourceMatch(pattern, resource string) bool {
+// Match function matches wild cards in 'pattern' for 'text'.
+func wildCardMatch(pattern, text string) bool {
 	if pattern == "" {
-		return resource == pattern
+		return text == pattern
 	}
 	if pattern == "*" {
 		return true
 	}
 	parts := strings.Split(pattern, "*")
 	if len(parts) == 1 {
-		return resource == pattern
+		return text == pattern
 	}
 	tGlob := strings.HasSuffix(pattern, "*")
 	end := len(parts) - 1
-	if !strings.HasPrefix(resource, parts[0]) {
+	if !strings.HasPrefix(text, parts[0]) {
 		return false
 	}
 	for i := 1; i < end; i++ {
-		if !strings.Contains(resource, parts[i]) {
+		if !strings.Contains(text, parts[i]) {
 			return false
 		}
-		idx := strings.Index(resource, parts[i]) + len(parts[i])
-		resource = resource[idx:]
+		idx := strings.Index(text, parts[i]) + len(parts[i])
+		text = text[idx:]
 	}
-	return tGlob || strings.HasSuffix(resource, parts[end])
+	return tGlob || strings.HasSuffix(text, parts[end])
+}
+
+// Match function matches wild cards in 'pattern' for resource.
+func resourceMatch(pattern, resource string) bool {
+	return wildCardMatch(pattern, resource)
+}
+
+// Match function matches wild cards in 'pattern' for action.
+func actionMatch(pattern, action string) bool {
+	return wildCardMatch(pattern, action)
 }
 
 // Verify if given resource matches with policy statement.
@@ -109,11 +115,11 @@ func bucketPolicyResourceMatch(resource string, statement policyStatement) bool
 		// the requested object can be given access based on the already set bucket policy if
 		// the match is successful.
 		// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html .
-		if matched := resourceMatch(resourcep, resource); !matched {
-			return false
+		if matched := resourceMatch(resourcep, resource); matched {
+			return true
 		}
 	}
-	return true
+	return false
 }
 
 // Verify if given condition matches with policy statement.
@@ -157,7 +163,6 @@ func bucketPolicyConditionMatch(conditions map[string]string, statement policySt
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
-
 	switch getRequestAuthType(r) {
 	default:
 		// For all unknown auth types return error.

bucket-policy-handlers_test.go

@@ -17,14 +17,18 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 )
 
 // Tests validate Bucket policy resource matcher.
 func TestBucketPolicyResourceMatch(t *testing.T) {
 
-	// generates\ statement with given resource..
+	// generates statement with given resource..
 	generateStatement := func(resource string) policyStatement {
 		statement := policyStatement{}
 		statement.Resources = []string{resource}
@@ -76,3 +80,592 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
 		}
 	}
 }
+
+// TestBucketPolicyActionMatch - Test validates whether given action on the
+// bucket/object matches the allowed actions in policyStatement.
+// This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
+// The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
+func TestBucketPolicyActionMatch(t *testing.T) {
+	bucketName := getRandomBucketName()
+	objectPrefix := "test-object"
+
+	testCases := []struct {
+		action         string
+		statement      policyStatement
+		expectedResult bool
+	}{
+		// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
+		// fetching of bucket location on an Anonymous/unsigned request.
+
+		//r ead-write bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 1).
+		{"s3:GetBucketLocation", getReadWriteBucketStatement(bucketName, objectPrefix), true},
+		//	write-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 2).
+		{"s3:GetBucketLocation", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
+		//	read-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 3).
+		{"s3:GetBucketLocation", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
+
+		// Any of the Object level access permissions shouldn't allow for GetBucketLocation operation  on an Anonymous/unsigned request (Test cases 4-6).
+		{"s3:GetBucketLocation", getReadWriteObjectStatement(bucketName, objectPrefix), false},
+		{"s3:GetBucketLocation", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+		{"s3:GetBucketLocation", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:ListBucketMultipartUploads is the action necessary to be present in the bucket policy to allow
+		// Listing of multipart uploads in a given bucket for an Anonymous/unsigned request.
+
+		//read-write bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 7).
+		{"s3:ListBucketMultipartUploads", getReadWriteBucketStatement(bucketName, objectPrefix), true},
+		//	write-only bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 8).
+		{"s3:ListBucketMultipartUploads", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
+		// read-only bucket policy is expected to not allow ListBucketMultipartUploads operation on an anonymous request (Test case 9).
+		// the allowed actions in read-only bucket statement are  "s3:GetBucketLocation","s3:ListBucket",
+		// this shouldnot allow for ListBucketMultipartUploads operations.
+		{"s3:ListBucketMultipartUploads", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Any of the object level policy will not allow for s3:ListBucketMultipartUploads (Test cases 10-12).
+		{"s3:ListBucketMultipartUploads", getReadWriteObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucketMultipartUploads", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucketMultipartUploads", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:ListBucket is the action necessary to be present in the bucket policy to allow
+		// listing of all objects inside a given bucket on an Anonymous/unsigned request.
+
+		// Cases for testing ListBucket access for different Bucket level access permissions.
+		// read-only bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 13).
+		{"s3:ListBucket", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
+		// read-write bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 14).
+		{"s3:ListBucket", getReadWriteBucketStatement(bucketName, objectPrefix), true},
+		// write-only  bucket policy is expected to not allow ListBucket operation on an anonymous request (Test case 15).
+		// the allowed actions in write-only  bucket statement are "s3:GetBucketLocation",	"s3:ListBucketMultipartUploads",
+		// this shouldnot allow for ListBucket operations.
+		{"s3:ListBucket", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing ListBucket access for different Object level access permissions (Test cases 16-18).
+		// Any of the Object level access permissions shouldn't allow for ListBucket operation  on an Anonymous/unsigned request.
+		{"s3:ListBucket", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucket", getReadWriteObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucket", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:DeleteObject is the action necessary to be present in the bucket policy to allow
+		// deleting/removal of  objects inside a given bucket for an Anonymous/unsigned request.
+
+		// Cases for testing DeleteObject access for different Bucket level access permissions (Test cases 19-21).
+		// Any of the Bucket level access permissions shouldn't allow for DeleteObject operation  on an Anonymous/unsigned request.
+		{"s3:DeleteObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:DeleteObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:DeleteObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing DeleteObject access for different Object level access permissions (Test cases 22).
+		// read-only bucket policy is expected to not allow Delete Object operation on an anonymous request.
+		{"s3:DeleteObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write bucket policy is expected to allow Delete Bucket operation on an anonymous request (Test cases 23).
+		{"s3:DeleteObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  bucket policy is expected to allow Delete Object operation on an anonymous request (Test cases 24).
+		{"s3:DeleteObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+
+		// s3:AbortMultipartUpload is the action necessary to be present in the bucket policy to allow
+		// cancelling or abortion of an already initiated multipart upload operation for an Anonymous/unsigned request.
+
+		// Cases for testing AbortMultipartUpload access for different Bucket level access permissions (Test cases 25-27).
+		// Any of the Bucket level access permissions shouldn't allow for AbortMultipartUpload operation on an Anonymous/unsigned request.
+		{"s3:AbortMultipartUpload", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:AbortMultipartUpload", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:AbortMultipartUpload", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing AbortMultipartUpload access for different Object level access permissions.
+		// read-only object policy is expected to not allow AbortMultipartUpload operation on an anonymous request (Test case 28).
+		{"s3:AbortMultipartUpload", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 29).
+		{"s3:AbortMultipartUpload", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 30).
+		{"s3:AbortMultipartUpload", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+
+		// s3:PutObject is the action necessary to be present in the bucket policy to allow
+		// uploading of an object for an Anonymous/unsigned request.
+
+		// Cases for testing PutObject access for different Bucket level access permissions (Test cases 31-33).
+		// Any of the Bucket level access permissions shouldn't allow for PutObject operation on an Anonymous/unsigned request.
+		{"s3:PutObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:PutObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:PutObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing PutObject access for different Object level access permissions.
+		// read-only object policy is expected to not allow PutObject operation on an anonymous request (Test case 34).
+		{"s3:PutObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write object policy is expected to allow PutObject operation on an anonymous request (Test case 35).
+		{"s3:PutObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  object policy is expected to allow PutObject operation on an anonymous request (Test case 36).
+		{"s3:PutObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+
+		// s3:GetObject is the action necessary to be present in the bucket policy to allow
+		// downloading of an object for an Anonymous/unsigned request.
+
+		// Cases for testing GetObject access for different Bucket level access permissions (Test cases 37-39).
+		// Any of the Bucket level access permissions shouldn't allow for GetObject operation on an Anonymous/unsigned request.
+		{"s3:GetObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:GetObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:GetObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing GetObject access for different Object level access permissions.
+		// read-only bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 40).
+		{"s3:GetObject", getReadOnlyObjectStatement(bucketName, objectPrefix), true},
+		// read-write bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 41).
+		{"s3:GetObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  bucket policy is expected to not allow  downloading of an Object on an anonymous request (Test case 42).
+		{"s3:GetObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:ListMultipartUploadParts is the action necessary to be present in the bucket policy to allow
+		// Listing of uploaded parts for an Anonymous/unsigned request.
+
+		// Any of the Bucket level access permissions shouldn't allow for ListMultipartUploadParts operation on an Anonymous/unsigned request.
+		// read-only bucket policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test cases 43-45).
+		{"s3:ListMultipartUploadParts", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:ListMultipartUploadParts", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:ListMultipartUploadParts", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// read-only object policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test case 46).
+		{"s3:ListMultipartUploadParts", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 47).
+		{"s3:ListMultipartUploadParts", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 48).
+		{"s3:ListMultipartUploadParts", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+	}
+	for i, testCase := range testCases {
+		actualResult := bucketPolicyActionMatch(testCase.action, testCase.statement)
+		if testCase.expectedResult != actualResult {
+			t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResult, actualResult)
+		}
+	}
+}
+
+// TestWildCardMatch - Tests validate the logic of wild card matching.
+// Its used to match the action and resources of the policy statement and the request.
+func TestWildCardMatch(t *testing.T) {
+	testCases := []struct {
+		pattern        string
+		text           string
+		expectedResult bool
+	}{
+		// Test case - 1.
+		// Test case with pattern "*". Expected to match any text.
+		{"*", "s3:GetObject", true},
+		// Test case - 2.
+		// Test case with empty pattern. This only matches empty string.
+		{"", "s3:GetObject", false},
+		// Test case - 3.
+		// Test case with empty pattern. This only matches empty string.
+		{"", "", true},
+		// Test case - 4.
+		// Test case with single "*" at the end.
+		{"s3:*", "s3:ListMultipartUploadParts", true},
+		// Test case - 5.
+		// Test case with a no "*". In this case the pattern and text should be the same.
+		{"s3:ListBucketMultipartUploads", "s3:ListBucket", false},
+		// Test case - 6.
+		// Test case with a no "*". In this case the pattern and text should be the same.
+		{"s3:ListBucket", "s3:ListBucket", true},
+		// Test case - 7.
+		// Test case with a no "*". In this case the pattern and text should be the same.
+		{"s3:ListBucketMultipartUploads", "s3:ListBucketMultipartUploads", true},
+		// Test case - 8.
+		// Test case with pattern containing key name with a prefix. Should accept the same text without a "*".
+		{"my-bucket/oo*", "my-bucket/oo", true},
+		// Test case - 9.
+		// Test case with "*" at the end of the pattern.
+		{"my-bucket/In*", "my-bucket/India/Karnataka/", true},
+		// Test case - 10.
+		// Test case with prefixes shuffled.
+		// This should fail.
+		{"my-bucket/In*", "my-bucket/Karnataka/India/", false},
+		// Test case - 11.
+		// Test case with text expanded to the wildcards in the pattern.
+		{"my-bucket/In*/Ka*/Ban", "my-bucket/India/Karnataka/Ban", true},
+		// Test case - 12.
+		// Test case with the  keyname part is repeated as prefix several times.
+		// This is valid.
+		{"my-bucket/In*/Ka*/Ban", "my-bucket/India/Karnataka/Ban/Ban/Ban/Ban/Ban", true},
+		// Test case - 13.
+		// Test case to validate that `*` can be expanded into multiple prefixes.
+		{"my-bucket/In*/Ka*/Ban", "my-bucket/India/Karnataka/Area1/Area2/Area3/Ban", true},
+		// Test case to validate that `*` can be expanded into multiple prefixes.
+		{"my-bucket/In*/Ka*/Ban", "my-bucket/India/State1/State2/Karnataka/Area1/Area2/Area3/Ban", true},
+		// Test case - 14.
+		// Test case where the keyname part of the pattern is expanded in the text.
+		{"my-bucket/In*/Ka*/Ban", "my-bucket/India/Karnataka/Bangalore", false},
+		// Test case - 15.
+		// Test case with prefixes and wildcard expanded for all "*".
+		{"my-bucket/In*/Ka*/Ban*", "my-bucket/India/Karnataka/Bangalore", true},
+		// Test case - 16.
+		// Test case with keyname part being a wildcard in the pattern.
+		{"my-bucket/*", "my-bucket/India", true},
+		// Test case - 17.
+		{"my-bucket/oo*", "my-bucket/odo", false},
+	}
+	// Iterating over the test cases, call the function under test and asert the output.
+	for i, testCase := range testCases {
+		actualResult := wildCardMatch(testCase.pattern, testCase.text)
+		if testCase.expectedResult != actualResult {
+			t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResult, actualResult)
+		}
+	}
+}
+
+// Wrapper for calling Put Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+func TestPutBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testPutBucketPolicyHandler)
+}
+
+// testPutBucketPolicyHandler - Test for Bucket policy end point.
+// TODO: Add exhaustive cases with various combination of statement fields.
+func testPutBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+	// Create bucket.
+	err := obj.MakeBucket(bucketName)
+	if err != nil {
+		// failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+	// Register the API end points with XL/FS object layer.
+	apiRouter := initTestAPIEndPoints(obj, []string{"PutBucketPolicy"})
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	credentials, rootPath, err := initTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	// template for constructing HTTP request body for PUT bucket policy.
+	bucketPolicyTemplate := `{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "s3:GetBucketLocation",
+                "s3:ListBucket"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s"
+            ]
+        },
+        {
+            "Action": [
+                "s3:GetObject"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s/this*"
+            ]
+        }
+    ]
+}`
+
+	// test cases with sample input and expected output.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, http.StatusNoContent},
+	}
+
+	// Iterating over the test cases, calling the function under test and asserting the response.
+	for i, testCase := range testCases {
+		// obtain the put bucket policy request body.
+		bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testCase.bucketName, testCase.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		req, err := newTestSignedRequest("PUT", getPutPolicyURL("", testCase.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code)
+		}
+	}
+}
+
+// Wrapper for calling Get Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+func TestGetBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testGetBucketPolicyHandler)
+}
+
+// testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket.
+// TODO: Add exhaustive cases with various combination of statement fields.
+func testGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+	// Create bucket.
+	err := obj.MakeBucket(bucketName)
+	if err != nil {
+		// failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+	// Register the API end points with XL/FS object layer.
+	// Registering only the PutBucketPolicy and GetBucketPolicy handlers.
+	apiRouter := initTestAPIEndPoints(obj, []string{"PutBucketPolicy", "GetBucketPolicy"})
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	credentials, rootPath, err := initTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	// template for constructing HTTP request body for PUT bucket policy.
+	bucketPolicyTemplate := `{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "s3:GetBucketLocation",
+                "s3:ListBucket"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s"
+            ]
+        },
+        {
+            "Action": [
+                "s3:GetObject"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s/this*"
+            ]
+        }
+    ]
+}`
+
+	// Writing bucket policy before running test on GetBucketPolicy.
+	putTestPolicies := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, http.StatusNoContent},
+	}
+
+	// Iterating over the cases and writing the bucket policy.
+	// its required to write the policies first before running tests on GetBucketPolicy.
+	for i, testPolicy := range putTestPolicies {
+		// obtain the put bucket policy request body.
+		bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		req, err := newTestSignedRequest("PUT", getPutPolicyURL("", testPolicy.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testPolicy.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, rec.Code)
+		}
+	}
+
+	// test cases with inputs and expected result for GetBucketPolicyHandler.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected output.
+		expectedBucketPolicy string
+		expectedRespStatus   int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, bucketPolicyTemplate, http.StatusOK},
+	}
+	// Iterating over the cases, fetching the policy and validating the response.
+	for i, testCase := range testCases {
+		// expected bucket policy json string.
+		expectedBucketPolicyStr := fmt.Sprintf(testCase.expectedBucketPolicy, testCase.bucketName, testCase.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		req, err := newTestSignedRequest("GET", getGetPolicyURL("", testCase.bucketName),
+			0, nil, testCase.accessKey, testCase.secretKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler, GetBucketPolicyHandler handles the request.
+		apiRouter.ServeHTTP(rec, req)
+		// Assert the response code with the expected status.
+		if rec.Code != testCase.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code)
+		}
+		// read the response body.
+		bucketPolicyReadBuf, err := ioutil.ReadAll(rec.Body)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Verify whether the bucket policy fetched is same as the one inserted.
+		if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+			t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
+		}
+	}
+}
+
+// Wrapper for calling Delete Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+func TestDeleteBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testDeleteBucketPolicyHandler)
+}
+
+// testDeleteBucketPolicyHandler - Test for Delete bucket policy end point.
+// TODO: Add exhaustive cases with various combination of statement fields.
+func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+	// Create bucket.
+	err := obj.MakeBucket(bucketName)
+	if err != nil {
+		// failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+	// Register the API end points with XL/FS object layer.
+	// Registering PutBucketPolicy and DeleteBucketPolicy handlers.
+	apiRouter := initTestAPIEndPoints(obj, []string{"PutBucketPolicy", "DeleteBucketPolicy"})
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	credentials, rootPath, err := initTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	// template for constructing HTTP request body for PUT bucket policy.
+	bucketPolicyTemplate := `{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "s3:GetBucketLocation",
+                "s3:ListBucket"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s"
+            ]
+        },
+        {
+            "Action": [
+                "s3:GetObject"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s/this*"
+            ]
+        }
+    ]
+}`
+
+	// Writing bucket policy before running test on DeleteBucketPolicy.
+	putTestPolicies := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, http.StatusNoContent},
+	}
+
+	// Iterating over the cases and writing the bucket policy.
+	// its required to write the policies first before running tests on GetBucketPolicy.
+	for i, testPolicy := range putTestPolicies {
+		// obtain the put bucket policy request body.
+		bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		req, err := newTestSignedRequest("PUT", getPutPolicyURL("", testPolicy.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testPolicy.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, rec.Code)
+		}
+	}
+	// testcases with input and expected output for DeleteBucketPolicyHandler.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, http.StatusNoContent},
+	}
+	// Iterating over the cases and deleting the bucket policy and then asserting response.
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for Delete bucket policy endpoint.
+		req, err := newTestSignedRequest("DELETE", getDeletePolicyURL("", testCase.bucketName),
+			0, nil, testCase.accessKey, testCase.secretKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler, DeleteBucketPolicyHandler  handles the request.
+		apiRouter.ServeHTTP(rec, req)
+		// Assert the response code with the expected status.
+		if rec.Code != testCase.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code)
+		}
+
+	}
+}

bucket-policy-parser.go

@@ -34,6 +34,8 @@ const (
 
 // supportedActionMap - lists all the actions supported by minio.
 var supportedActionMap = map[string]struct{}{
+	"*":                             {},
+	"s3:*":                          {},
 	"s3:GetObject":                  {},
 	"s3:ListBucket":                 {},
 	"s3:PutObject":                  {},
@@ -255,7 +257,7 @@ func checkBucketPolicyResources(bucket string, bucketPolicy BucketPolicy) APIErr
 		for _, otherResource := range resources {
 			// Common prefix reject such rules.
 			if strings.HasPrefix(otherResource, resource) {
-				return ErrMalformedPolicy
+				return ErrPolicyNesting
 			}
 		}
 	}

bucket-policy-parser_test.go

@@ -69,60 +69,89 @@ var (
 	}
 )
 
-// Obtain statements for read-write BucketPolicy.
-func setReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
-	bucketResourceStatement := policyStatement{}
+// Obtain bucket statement for read-write BucketPolicy.
+func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatement {
 	objectResourceStatement := policyStatement{}
-	statements := []policyStatement{}
-
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal.AWS = []string{"*"}
-	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
-	bucketResourceStatement.Actions = readWriteBucketActions
 	objectResourceStatement.Effect = "Allow"
 	objectResourceStatement.Principal.AWS = []string{"*"}
 	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
 	objectResourceStatement.Actions = readWriteObjectActions
+	return objectResourceStatement
+}
+
+// Obtain object statement for read-write BucketPolicy.
+func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatement {
+	bucketResourceStatement := policyStatement{}
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal.AWS = []string{"*"}
+	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
+	bucketResourceStatement.Actions = readWriteBucketActions
+	return bucketResourceStatement
+}
+
+// Obtain statements for read-write BucketPolicy.
+func getReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
 	// Save the read write policy.
-	statements = append(statements, bucketResourceStatement, objectResourceStatement)
+	statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
 	return statements
 }
 
-// Obtain statements for read only BucketPolicy.
-func setReadOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+// Obtain bucket statement for read only BucketPolicy.
+func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
 	bucketResourceStatement := policyStatement{}
-	objectResourceStatement := policyStatement{}
-	statements := []policyStatement{}
-
 	bucketResourceStatement.Effect = "Allow"
 	bucketResourceStatement.Principal.AWS = []string{"*"}
 	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
 	bucketResourceStatement.Actions = readOnlyBucketActions
+	return bucketResourceStatement
+}
+
+// Obtain object statement for read only BucketPolicy.
+func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
 	objectResourceStatement.Effect = "Allow"
 	objectResourceStatement.Principal.AWS = []string{"*"}
 	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
 	objectResourceStatement.Actions = readOnlyObjectActions
+	return objectResourceStatement
+}
+
+// Obtain statements for read only BucketPolicy.
+func getReadOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
 	// Save the read only policy.
-	statements = append(statements, bucketResourceStatement, objectResourceStatement)
+	statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
 	return statements
 }
 
-// Obtain statements for write only BucketPolicy.
-func setWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+// Obtain bucket statements for write only BucketPolicy.
+func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
+
 	bucketResourceStatement := policyStatement{}
-	objectResourceStatement := policyStatement{}
-	statements := []policyStatement{}
-	// Write only policy.
 	bucketResourceStatement.Effect = "Allow"
 	bucketResourceStatement.Principal.AWS = []string{"*"}
 	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
 	bucketResourceStatement.Actions = writeOnlyBucketActions
+	return bucketResourceStatement
+}
+
+// Obtain object statements for write only BucketPolicy.
+func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
 	objectResourceStatement.Effect = "Allow"
 	objectResourceStatement.Principal.AWS = []string{"*"}
 	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
 	objectResourceStatement.Actions = writeOnlyObjectActions
+	return objectResourceStatement
+}
+
+// Obtain statements for write only BucketPolicy.
+func getWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
+	// Write only policy.
 	// Save the write only policy.
-	statements = append(statements, bucketResourceStatement, objectResourceStatement)
+	statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
 	return statements
 }
 
@@ -145,11 +174,13 @@ func TestIsValidActions(t *testing.T) {
 		{[]string{}, errors.New("Action list cannot be empty."), false},
 		// Test case - 3.
 		// "s3:DeleteEverything"" is an invalid Action.
-		{[]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}, errors.New("Unsupported action found: ‘s3:DeleteEverything’, please validate your policy document."), false},
+		{[]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"},
+			errors.New("Unsupported action found: ‘s3:DeleteEverything’, please validate your policy document."), false},
 
 		// Inputs with valid Action.
 		// Test Case - 4.
-		{[]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts"}, nil, true},
+		{[]string{"s3:*", "*", "s3:GetObject", "s3:ListBucket",
+			"s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts"}, nil, true},
 	}
 	for i, testCase := range testCases {
 		err := isValidActions(testCase.actions)
@@ -464,31 +495,31 @@ func TestCheckBucketPolicyResources(t *testing.T) {
 	bucketAccessPolicies := []BucketPolicy{
 		// BucketPolicy - 1.
 		// Contains valid read only policy statement.
-		{Version: "1.0", Statements: setReadOnlyStatement("minio-bucket", "")},
+		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
 		// BucketPolicy - 2.
 		// Contains valid read-write only policy statement.
-		{Version: "1.0", Statements: setReadWriteStatement("minio-bucket", "Asia/")},
+		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
 		// BucketPolicy - 3.
 		// Contains valid write only policy statement.
-		{Version: "1.0", Statements: setWriteOnlyStatement("minio-bucket", "Asia/India/")},
+		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
 		// BucketPolicy - 4.
 		// Contains invalidPrefixActions.
 		// Since resourcePrefix is not to the bucket-name, it return ErrMalformedPolicy.
-		{Version: "1.0", Statements: setReadOnlyStatement("minio-bucket-fail", "Asia/India/")},
+		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket-fail", "Asia/India/")},
 		// BucketPolicy - 5.
 		// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
 		// but bucket part of the resource is not equal to the bucket name.
 		// this results in return of ErrMalformedPolicy.
-		{Version: "1.0", Statements: setValidPrefixActions(setWriteOnlyStatement("minio-bucket-fail", "Asia/India/"))},
+		{Version: "1.0", Statements: setValidPrefixActions(getWriteOnlyStatement("minio-bucket-fail", "Asia/India/"))},
 		// BucketPolicy - 6.
 		// contructing policy statement with recursive resources.
 		// should result in ErrMalformedPolicy
-		{Version: "1.0", Statements: setRecurseResource(setValidPrefixActions(setWriteOnlyStatement("minio-bucket", "")))},
+		{Version: "1.0", Statements: setRecurseResource(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "")))},
 		// BucketPolciy - 7.
 		// constructing policy statement with non recursive but
 		// lexically close resources.
 		// should result in ErrNone.
-		{Version: "1.0", Statements: setResourceLexical(setValidPrefixActions(setWriteOnlyStatement("minio-bucket", "oo")))},
+		{Version: "1.0", Statements: setResourceLexical(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "oo")))},
 	}
 
 	testCases := []struct {
@@ -514,8 +545,8 @@ func TestCheckBucketPolicyResources(t *testing.T) {
 		{bucketAccessPolicies[4], ErrMalformedPolicy, false},
 		// Test case - 6.
 		// contructing policy statement with recursive resources.
-		// should result in ErrMalformedPolicy.
-		{bucketAccessPolicies[5], ErrMalformedPolicy, false},
+		// should result in ErrPolicyNesting.
+		{bucketAccessPolicies[5], ErrPolicyNesting, false},
 		// Test case - 7.
 		// constructing policy statement with lexically close
 		// characters.
@@ -575,25 +606,25 @@ func TestParseBucketPolicy(t *testing.T) {
 		{Version: "", Statements: []policyStatement{}},
 		// BucketPolicy - 2.
 		// Readonly BucketPolicy.
-		{Version: "1.0", Statements: setReadOnlyStatement("minio-bucket", "")},
+		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
 		// BucketPolicy - 3.
 		// Read-Write bucket policy.
-		{Version: "1.0", Statements: setReadWriteStatement("minio-bucket", "Asia/")},
+		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
 		// BucketPolicy - 4.
 		// Write only bucket policy.
-		{Version: "1.0", Statements: setWriteOnlyStatement("minio-bucket", "Asia/India/")},
+		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
 		// BucketPolicy - 5.
 		// BucketPolicy statement contains unsupported action.
-		{Version: "1.0", Statements: setUnsupportedActions(setReadOnlyStatement("minio-bucket", ""))},
+		{Version: "1.0", Statements: setUnsupportedActions(getReadOnlyStatement("minio-bucket", ""))},
 		// BucketPolicy - 6.
 		// BucketPolicy statement contains unsupported Effect.
-		{Version: "1.0", Statements: setUnsupportedEffect(setReadWriteStatement("minio-bucket", "Asia/"))},
+		{Version: "1.0", Statements: setUnsupportedEffect(getReadWriteStatement("minio-bucket", "Asia/"))},
 		// BucketPolicy - 7.
 		// BucketPolicy statement contains unsupported Principal.
-		{Version: "1.0", Statements: setUnsupportedPrincipals(setWriteOnlyStatement("minio-bucket", "Asia/India/"))},
+		{Version: "1.0", Statements: setUnsupportedPrincipals(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
 		// BucketPolicy - 8.
 		// BucketPolicy statement contains unsupported Resource.
-		{Version: "1.0", Statements: setUnsupportedResources(setWriteOnlyStatement("minio-bucket", "Asia/India/"))},
+		{Version: "1.0", Statements: setUnsupportedResources(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
 	}
 
 	testCases := []struct {

certs.go

@@ -19,8 +19,6 @@ package main
 import (
 	"os"
 	"path/filepath"
-
-	"github.com/minio/go-homedir"
 )
 
 // createCertsPath create certs path.
@@ -34,11 +32,12 @@ func createCertsPath() error {
 
 // getCertsPath get certs path.
 func getCertsPath() (string, error) {
-	homeDir, err := homedir.Dir()
+	var certsPath string
+	configDir, err := getConfigPath()
 	if err != nil {
 		return "", err
 	}
-	certsPath := filepath.Join(homeDir, globalMinioCertsDir)
+	certsPath = filepath.Join(configDir, globalMinioCertsDir)
 	return certsPath, nil
 }
 

checkport.go

@@ -0,0 +1,93 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"net"
+	"os"
+	"syscall"
+)
+
+// Make sure that none of the other processes are listening on the
+// specified port on any of the interfaces.
+//
+// On linux if a process is listening on 127.0.0.1:9000 then Listen()
+// on ":9000" fails with the error "port already in use".
+// However on Mac OSX Listen() on ":9000" falls back to the IPv6 address.
+// This causes confusion on Mac OSX that minio server is not reachable
+// on 127.0.0.1 even though minio server is running. So before we start
+// the minio server we make sure that the port is free on all the IPs.
+func checkPortAvailability(port int) {
+	isAddrInUse := func(err error) bool {
+		// Check if the syscall error is EADDRINUSE.
+		// EADDRINUSE is the system call error if another process is
+		// already listening at the specified port.
+		neterr, ok := err.(*net.OpError)
+		if !ok {
+			return false
+		}
+		osErr, ok := neterr.Err.(*os.SyscallError)
+		if !ok {
+			return false
+		}
+		sysErr, ok := osErr.Err.(syscall.Errno)
+		if !ok {
+			return false
+		}
+		if sysErr != syscall.EADDRINUSE {
+			return false
+		}
+		return true
+	}
+	ifcs, err := net.Interfaces()
+	if err != nil {
+		fatalIf(err, "Unable to list interfaces.")
+	}
+	for _, ifc := range ifcs {
+		addrs, err := ifc.Addrs()
+		if err != nil {
+			fatalIf(err, "Unable to list addresses on interface %s.", ifc.Name)
+		}
+		for _, addr := range addrs {
+			ipnet, ok := addr.(*net.IPNet)
+			if !ok {
+				errorIf(errors.New(""), "Failed to assert type on (*net.IPNet) interface.")
+				continue
+			}
+			ip := ipnet.IP
+			network := "tcp4"
+			if ip.To4() == nil {
+				network = "tcp6"
+			}
+			tcpAddr := net.TCPAddr{IP: ip, Port: port, Zone: ifc.Name}
+			l, err := net.ListenTCP(network, &tcpAddr)
+			if err != nil {
+				if isAddrInUse(err) {
+					// Fail if port is already in use.
+					fatalIf(err, "Unable to listen on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
+				} else {
+					// Ignore other errors.
+					continue
+				}
+			}
+			if err = l.Close(); err != nil {
+				fatalIf(err, "Unable to close listener on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
+			}
+		}
+	}
+}

config-migrate.go

@@ -49,7 +49,7 @@ func purgeV1() {
 		fatalIf(err, "Unable to retrieve config path.")
 
 		configFile := filepath.Join(configPath, "fsUsers.json")
-		os.RemoveAll(configFile)
+		removeAll(configFile)
 	}
 	fatalIf(errors.New(""), "Failed to migrate unrecognized config version ‘"+cv1.Version+"’.")
 }

dist/benchmark/benchcmp.sh

@@ -0,0 +1,67 @@
+#!/bin/bash          
+
+# usage: ./benchcmp.sh <commit-sha1> <commit-sha2>
+# Exit on any non zero return value on execution of a command. 
+set -e 
+
+# path of benchcmp. 
+benchcmp=${GOPATH}/bin/benchcmp
+
+# function which runs the benchmark comparison. 
+RunBenchCmp () {
+	# Path for storing output of benchmark at commit 1. 
+	commit1Bench=/tmp/minio-$1.bench
+	# Path for storing output of benchmark at commit 2. 
+	commit2Bench=/tmp/minio-$2.bench
+	# switch to commit $1. 
+	git checkout $1
+	# Check if the benchmark results for given commit 1 already exists. 
+	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
+	if [[ ! -f $commit1Bench  ]]
+	then
+
+		echo "Running benchmarks at $1"
+		go test -run=NONE -bench=. | tee $commit1Bench 
+	fi
+	# get back to the commit from which it was started. 
+	git checkout -
+	echo "Checking into commit $2"
+	# switch to commit $2 
+	git checkout $2
+	# Check if the benchmark results for given commit 2 already exists. 
+	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
+	if [[ ! -f $commit2Bench  ]]
+	then
+	# Running benchmarks at $2.
+		echo "Running benchmarks at $2"
+		go test -run=NONE -bench=. | tee $commit2Bench 
+	fi
+	
+	# get back to the commit from which it was started. 
+	git checkout - 
+	# Comparing the benchmarks.
+	echo "Running benchmark comparison between $1 and $2 ..."
+	$benchcmp $commit1Bench $commit2Bench 
+	echo "Done."
+}
+
+# check if 2 commit SHA's of snapshots of code for which benchmp has to be done is provided. 
+if [ ! $# -eq 2 ]
+then
+	# exit if commit SHA's are not provided. 
+	echo $#
+	echo "Need Commit SHA's of 2 snapshots to be supplied to run benchmark comparision."
+	exit 1
+fi
+
+# check if benchcmp exists. 
+if [[ -x "$benchcmp" ]]
+then
+	RunBenchCmp $1 $2
+else
+	# install benchcmp if doesnt't exist. 
+	echo "fetching Benchcmp..."
+	go get -u golang.org/x/tools/cmd/benchcmp
+	echo "Done."
+	RunBenchCmp $1 $2
+fi

dist/linux-systemd/minio.service

@@ -0,0 +1,50 @@
+[Unit]
+Description=Minio
+Documentation=http://minio.io
+Wants=network-online.target
+After=network-online.target
+AssertPathExists=/etc/minio/
+AssertFileIsExecutable=/usr/share/minio/bin/minio
+
+[Service]
+WorkingDirectory=/usr/share/minio/
+
+User=minio
+Group=minio
+
+PermissionsStartOnly=true
+    
+EnvironmentFile=-/etc/default/minio
+ExecStartPre=/bin/bash -c "[ -n \"${MINIO_VOLUMES}\" ] || echo \"Variable MINIO_VOLUMES not set in /etc/defaults/minio\""
+ExecStartPre=/bin/bash -c "(for i in $MINIO_VOLUMES; do if [ ! -d $i ]; then echo \"Directory $i of variable MINIO_VOLUMES isn't an existing directory\" && false; exit; fi; done)"
+
+ExecStartPre=-/bin/chown -R root:minio /etc/minio/
+ExecStartPre=-/bin/chmod -R 660 /etc/minio/
+ExecStartPre=-/bin/chmod -R ug+X /etc/minio/
+
+ExecStart=/usr/share/minio/bin/minio \
+                --config-dir "/etc/minio/" \
+                server \
+                $MINIO_OPTS \
+                $MINIO_VOLUMES
+
+StandardOutput=journal
+StandardError=inherit
+
+# Specifies the maximum file descriptor number that can be opened by this process
+LimitNOFILE=65536
+
+# Disable timeout logic and wait until process is stopped
+TimeoutStopSec=0
+
+# SIGTERM signal is used to stop Minio
+KillSignal=SIGTERM
+
+SendSIGKILL=no
+
+SuccessExitStatus=0
+
+[Install]
+WantedBy=multi-user.target
+
+# Built for ${project.name}-${project.version} (${project.name})

dist/linux-systemd/readme.md

@@ -0,0 +1,35 @@
+# Readme
+Service script for minio service for systemd.
+
+# Installation
+```
+mkdir /etc/minio/
+mkdir -p /usr/share/minio/bin/
+wget -o /usr/share/minio/bin/minio http://...
+```
+
+Create minio user.
+```
+useradd minio
+```
+
+Create default configuration. Don't forget to update MINIO_VOLUMES with the correct path(s).
+```
+cat <<EOT >> /etc/default/minio
+MINIO_OPTS="--address :9000"
+MINIO_VOLUMES="/tmp/minio/"
+EOT
+```
+
+# Systemctl
+
+Put minio.service in /etc/systemd/system/
+```
+curl https://raw.githubusercontent.com/minio/minio/master/dist/linux-systemd/minio.service > /etc/systemd/system/
+```
+
+Enable startup on boot
+```
+systemctl enable minio.service
+```
+

docker/Dockerfile

@@ -1,10 +0,0 @@
-# use "make dockerimage" to build
-FROM alpine:3.3
-RUN apk add --no-cache ca-certificates bash
-ADD minio.dockerimage /minio
-ADD export /export
-EXPOSE 9000
-COPY start.sh /start.sh
-ENV ALLOW_CONTAINER_ROOT=1
-ENTRYPOINT ["/start.sh"]
-CMD ["/export"]

docker/start.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-## Fix this check when we arrive at XL.
-if [ -z "$1" ]; then
-    echo "Invalid arguments"
-    echo "Usage: <export_dir>"
-    exit 1
-else
-    /bin/mkdir -p "$1" && /minio server "$1"
-fi

docs/backend/fs/fs.json

@@ -1,5 +1,5 @@
 {
-    "version": "1",
+    "version": "1.0.0",
     "format": "fs",
     "minio": {
         "release": "DEVELOPMENT.GOGET"
@@ -7,7 +7,7 @@
     "parts": [
         {
             "number": 1,
-            "name": "object1",
+            "name": "part.1",
             "size": 29,
             "eTag": "",
         },

docs/backend/xl/uploads.json

@@ -1,5 +1,5 @@
 {
-    "version": "1",
+    "version": "1.0.0",
     "format": "xl",
     "uploadIds": [
         {

docs/backend/xl/xl.json

@@ -4,19 +4,13 @@
             "number": 1,
             "size": 5242880,
             "etag": "3565c6e741e69a007a5ac7db893a62b5",
-            "name": "object1"
+            "name": "part.1"
         },
         {
             "number": 2,
             "size": 5242880,
             "etag": "d416712335c280ab1e39498552937764",
-            "name": "object2"
-        },
-        {
-            "number": 3,
-            "size": 4338324,
-            "etag": "8a98c5c54d81c6c95ed9bdcaeb941aaf",
-            "name": "object3"
+            "name": "part.2"
         }
     ],
     "meta": {
@@ -36,22 +30,21 @@
         "data": 5,
         "checksum": [
             {
-                "name": "object1",
-                "algorithm": "sha512",
-                "hash": "d9910e1492446389cfae6fe979db0245f96ca97ca2c7a25cab45805882004479320d866a47ea1f7be6a62625dd4de6caf7816009ef9d62779346d01a221b335c",
+                "name": "part.1",
+                "algorithm": "blake2b",
+                "hash": "173c2cda7fb9e2798f91ba10135b65475fd2b97684355b328941d1c470a37549502672b2b4630c8268d5e5f79c8eb2fe433a25c6368f31d75b2c0504e3104c0e",
             },
             {
-                "name": "object2",
-                "algorithm": "sha512",
-                "hash": "d9910e1492446389cfae6fe979db0245f96ca97ca2c7a25cab45805882004479320d866a47ea1f7be6a62625dd4de6caf7816009ef9d62779346d01a221b335c",
+                "name": "part.2",
+                "algorithm": "blake2b",
+                "hash": "14f9ba0006b2db7cd171507e79032ad3a34eab7df02b6564c487614b6d1e6613343a4f56074aa3473ac9c3b26e00cbbeae937d478cca4cfb138e72838ebb0826",
             },
         ],
     },
     "stat": {
-        "version": 0,
         "modTime": "2016-05-24T00:09:40.122390255Z",
         "size": 14824084
     },
     "format": "xl",
-    "version": "1"
+    "version": "1.0.0"
 }

docs/caching.md

@@ -0,0 +1,39 @@
+## Object caching
+
+Object caching by turned on by default with following settings
+
+  - Default cache size 8GB, can be changed from environment variable
+    ``MINIO_CACHE_SIZE`` supports both SI and ISO IEC standard forms
+    for input size parameters.
+
+  - Default expiration of entries is 72 hours, can be changed from
+    environment variable ``MINIO_CACHE_EXPIRY`` supportings Go
+    ``time.Duration`` with valid units  "ns", "us" (or "µs"),
+    "ms", "s", "m", "h".
+
+  - Default expiry interval is 1/4th of the expiration hours, so
+    expiration sweep happens across the cache every 1/4th the time
+    duration of the set entry expiration duration.
+
+### Tricks
+
+Setting MINIO_CACHE_SIZE=0 will turn off caching entirely.
+Setting MINIO_CACHE_EXPIRY=0s will turn off cache garbage collections,
+all cached objects will never expire.
+
+### Behavior
+
+Caching happens for both GET and PUT.
+
+- GET caches new objects for entries not found in cache,
+otherwise serves from the cache.
+
+- PUT/POST caches all successfully uploaded objects.
+
+NOTE: Cache is not populated if there are any errors
+      while reading from the disk.
+
+Expiration happens automatically based on the configured
+interval as explained above, frequently accessed objects
+stay alive for significantly longer time due to the fact
+that expiration time is reset for every cache hit.

erasure-createfile.go

@@ -28,34 +28,53 @@ import (
 // erasureCreateFile - writes an entire stream by erasure coding to
 // all the disks, writes also calculate individual block's checksum
 // for future bit-rot protection.
-func erasureCreateFile(disks []StorageAPI, volume string, path string, partName string, data io.Reader, eInfos []erasureInfo) (newEInfos []erasureInfo, size int64, err error) {
-	// Allocated blockSized buffer for reading.
-	buf := make([]byte, blockSizeV1)
-	hashWriters := newHashWriters(len(disks))
-
+func erasureCreateFile(disks []StorageAPI, volume string, path string, partName string, data io.Reader, eInfos []erasureInfo, writeQuorum int) (newEInfos []erasureInfo, size int64, err error) {
 	// Just pick one eInfo.
 	eInfo := pickValidErasureInfo(eInfos)
 
+	// Allocated blockSized buffer for reading.
+	buf := make([]byte, eInfo.BlockSize)
+	hashWriters := newHashWriters(len(disks))
+
 	// Read until io.EOF, erasure codes data and writes to all disks.
 	for {
-		var n int
-		n, err = io.ReadFull(data, buf)
-		if err == io.EOF {
+		var blocks [][]byte
+		n, rErr := io.ReadFull(data, buf)
+		// FIXME: this is a bug in Golang, n == 0 and err ==
+		// io.ErrUnexpectedEOF for io.ReadFull function.
+		if n == 0 && rErr == io.ErrUnexpectedEOF {
+			return nil, 0, rErr
+		}
+		if rErr == io.EOF {
+			// We have reached EOF on the first byte read, io.Reader
+			// must be 0bytes, we don't need to erasure code
+			// data. Will create a 0byte file instead.
+			if size == 0 {
+				blocks = make([][]byte, len(disks))
+				rErr = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum)
+				if rErr != nil {
+					return nil, 0, rErr
+				}
+			} // else we have reached EOF after few reads, no need to
+			// add an additional 0bytes at the end.
 			break
 		}
-		if err != nil && err != io.ErrUnexpectedEOF {
-			return nil, 0, err
+		if rErr != nil && rErr != io.ErrUnexpectedEOF {
+			return nil, 0, rErr
 		}
-		size += int64(n)
-		var blocks [][]byte
-		// Returns encoded blocks.
-		blocks, err = encodeData(buf[:n], eInfo.DataBlocks, eInfo.ParityBlocks)
-		if err != nil {
-			return nil, 0, err
-		}
-		err = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters)
-		if err != nil {
-			return nil, 0, err
+		if n > 0 {
+			// Returns encoded blocks.
+			var enErr error
+			blocks, enErr = encodeData(buf[0:n], eInfo.DataBlocks, eInfo.ParityBlocks)
+			if enErr != nil {
+				return nil, 0, enErr
+			}
+
+			// Write to all disks.
+			if err = appendFile(disks, volume, path, blocks, eInfo.Distribution, hashWriters, writeQuorum); err != nil {
+				return nil, 0, err
+			}
+			size += int64(n)
 		}
 	}
 
@@ -65,7 +84,7 @@ func erasureCreateFile(disks []StorageAPI, volume string, path string, partName
 		blockIndex := eInfo.Distribution[index] - 1
 		checkSums[blockIndex] = checkSumInfo{
 			Name:      partName,
-			Algorithm: "sha512",
+			Algorithm: "blake2b",
 			Hash:      hex.EncodeToString(hashWriters[blockIndex].Sum(nil)),
 		}
 	}
@@ -109,7 +128,7 @@ func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, erro
 }
 
 // appendFile - append data buffer at path.
-func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, distribution []int, hashWriters []hash.Hash) (err error) {
+func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, distribution []int, hashWriters []hash.Hash, writeQuorum int) (err error) {
 	var wg = &sync.WaitGroup{}
 	var wErrs = make([]error, len(disks))
 	// Write encoded data to quorum disks in parallel.
@@ -123,15 +142,11 @@ func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, dist
 			defer wg.Done()
 			// Pick the block from the distribution.
 			blockIndex := distribution[index] - 1
-			n, wErr := disk.AppendFile(volume, path, enBlocks[blockIndex])
+			wErr := disk.AppendFile(volume, path, enBlocks[blockIndex])
 			if wErr != nil {
 				wErrs[index] = wErr
 				return
 			}
-			if n != int64(len(enBlocks[blockIndex])) {
-				wErrs[index] = errUnexpected
-				return
-			}
 
 			// Calculate hash for each blocks.
 			hashWriters[blockIndex].Write(enBlocks[blockIndex])
@@ -144,6 +159,9 @@ func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, dist
 	// Wait for all the appends to finish.
 	wg.Wait()
 
-	// Return success.
+	// Do we have write quorum?.
+	if !isDiskQuorum(wErrs, writeQuorum) {
+		return toObjectErr(errXLWriteQuorum, volume, path)
+	}
 	return nil
 }

erasure-readfile.go

@@ -17,116 +17,295 @@
 package main
 
 import (
+	"bytes"
 	"encoding/hex"
 	"errors"
+	"io"
+	"sync"
 
 	"github.com/klauspost/reedsolomon"
 )
 
-// erasureReadFile - read an entire erasure coded file at into a byte
-// array. Erasure coded parts are often few mega bytes in size and it
-// is convenient to return them as byte slice. This function also
-// supports bit-rot detection by verifying checksum of individual
-// block's checksum.
-func erasureReadFile(disks []StorageAPI, volume string, path string, partName string, size int64, eInfos []erasureInfo) ([]byte, error) {
-	// Return data buffer.
-	var buffer []byte
+// isSuccessDecodeBlocks - do we have all the blocks to be
+// successfully decoded?. Input encoded blocks ordered matrix.
+func isSuccessDecodeBlocks(enBlocks [][]byte, dataBlocks int) bool {
+	// Count number of data and parity blocks that were read.
+	var successDataBlocksCount = 0
+	var successParityBlocksCount = 0
+	for index := range enBlocks {
+		if enBlocks[index] == nil {
+			continue
+		}
+		// block index lesser than data blocks, update data block count.
+		if index < dataBlocks {
+			successDataBlocksCount++
+			continue
+		} // else { // update parity block count.
+		successParityBlocksCount++
+	}
+	// Returns true if we have atleast dataBlocks + 1 parity.
+	return successDataBlocksCount == dataBlocks || successDataBlocksCount+successParityBlocksCount >= dataBlocks+1
+}
 
-	// Total size left
-	totalSizeLeft := size
+// isSuccessDataBlocks - do we have all the data blocks?
+// Input encoded blocks ordered matrix.
+func isSuccessDataBlocks(enBlocks [][]byte, dataBlocks int) bool {
+	// Count number of data blocks that were read.
+	var successDataBlocksCount = 0
+	for index := range enBlocks[:dataBlocks] {
+		if enBlocks[index] == nil {
+			continue
+		}
+		// block index lesser than data blocks, update data block count.
+		if index < dataBlocks {
+			successDataBlocksCount++
+		}
+	}
+	// Returns true if we have atleast the dataBlocks.
+	return successDataBlocksCount >= dataBlocks
+}
 
-	// Starting offset for reading.
-	startOffset := int64(0)
+// getOrderedDisks - get ordered disks from erasure distribution.
+// returns ordered slice of disks from their actual distribution.
+func getOrderedDisks(distribution []int, disks []StorageAPI, blockCheckSums []checkSumInfo) (orderedDisks []StorageAPI, orderedBlockCheckSums []checkSumInfo) {
+	orderedDisks = make([]StorageAPI, len(disks))
+	orderedBlockCheckSums = make([]checkSumInfo, len(disks))
+	// From disks gets ordered disks.
+	for index := range disks {
+		blockIndex := distribution[index]
+		orderedDisks[blockIndex-1] = disks[index]
+		orderedBlockCheckSums[blockIndex-1] = blockCheckSums[index]
+	}
+	return orderedDisks, orderedBlockCheckSums
+}
 
-	// Gather previously calculated block checksums.
-	blockCheckSums := metaPartBlockChecksums(disks, eInfos, partName)
+// Return readable disks slice from which we can read parallelly.
+func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDisks []StorageAPI, nextIndex int, err error) {
+	readDisks = make([]StorageAPI, len(orderedDisks))
+	dataDisks := 0
+	parityDisks := 0
+	// Count already read data and parity chunks.
+	for i := 0; i < index; i++ {
+		if orderedDisks[i] == nil {
+			continue
+		}
+		if i < dataBlocks {
+			dataDisks++
+		} else {
+			parityDisks++
+		}
+	}
+
+	// Sanity checks - we should never have this situation.
+	if dataDisks == dataBlocks {
+		return nil, 0, errUnexpected
+	}
+	if dataDisks+parityDisks >= dataBlocks+1 {
+		return nil, 0, errUnexpected
+	}
+
+	// Find the disks from which next set of parallel reads should happen.
+	for i := index; i < len(orderedDisks); i++ {
+		if orderedDisks[i] == nil {
+			continue
+		}
+		if i < dataBlocks {
+			dataDisks++
+		} else {
+			parityDisks++
+		}
+		readDisks[i] = orderedDisks[i]
+		if dataDisks == dataBlocks {
+			return readDisks, i + 1, nil
+		}
+		if dataDisks+parityDisks == dataBlocks+1 {
+			return readDisks, i + 1, nil
+		}
+	}
+	return nil, 0, errXLReadQuorum
+}
+
+// parallelRead - reads chunks in parallel from the disks specified in []readDisks.
+func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []StorageAPI, enBlocks [][]byte, blockOffset int64, curChunkSize int64, bitRotVerify func(diskIndex int) bool) {
+	// WaitGroup to synchronise the read go-routines.
+	wg := &sync.WaitGroup{}
+
+	// Read disks in parallel.
+	for index := range readDisks {
+		if readDisks[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		// Reads chunk from readDisk[index] in routine.
+		go func(index int) {
+			defer wg.Done()
+
+			// Verify bit rot for the file on this disk.
+			if !bitRotVerify(index) {
+				// So that we don't read from this disk for the next block.
+				orderedDisks[index] = nil
+				return
+			}
+
+			// Chunk writer.
+			chunkWriter := bytes.NewBuffer(make([]byte, 0, curChunkSize))
+
+			// CopyN - copies until current chunk size.
+			err := copyN(chunkWriter, readDisks[index], volume, path, blockOffset, curChunkSize)
+			if err != nil {
+				// So that we don't read from this disk for the next block.
+				orderedDisks[index] = nil
+				return
+			}
+
+			// Copy the read blocks.
+			enBlocks[index] = chunkWriter.Bytes()
+
+			// Successfully read.
+		}(index)
+	}
+
+	// Waiting for first routines to finish.
+	wg.Wait()
+}
+
+// erasureReadFile - read bytes from erasure coded files and writes to given writer.
+// Erasure coded files are read block by block as per given erasureInfo and data chunks
+// are decoded into a data block. Data block is trimmed for given offset and length,
+// then written to given writer. This function also supports bit-rot detection by
+// verifying checksum of individual block's checksum.
+func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path string, partName string, eInfos []erasureInfo, offset int64, length int64, totalLength int64) (int64, error) {
+	// Offset and length cannot be negative.
+	if offset < 0 || length < 0 {
+		return 0, errUnexpected
+	}
 
 	// Pick one erasure info.
 	eInfo := pickValidErasureInfo(eInfos)
 
-	// Write until each parts are read and exhausted.
-	for totalSizeLeft > 0 {
-		// Calculate the proper block size.
-		var curBlockSize int64
-		if eInfo.BlockSize < totalSizeLeft {
-			curBlockSize = eInfo.BlockSize
-		} else {
-			curBlockSize = totalSizeLeft
+	// Gather previously calculated block checksums.
+	blockCheckSums := metaPartBlockChecksums(disks, eInfos, partName)
+
+	// []orderedDisks will have first eInfo.DataBlocks disks as data
+	// disks and rest will be parity.
+	orderedDisks, orderedBlockCheckSums := getOrderedDisks(eInfo.Distribution, disks, blockCheckSums)
+
+	// bitRotVerify verifies if the file on a particular disk doesn't have bitrot
+	// by verifying the hash of the contents of the file.
+	bitRotVerify := func() func(diskIndex int) bool {
+		verified := make([]bool, len(orderedDisks))
+		// Return closure so that we have reference to []verified and
+		// not recalculate the hash on it every time the function is
+		// called for the same disk.
+		return func(diskIndex int) bool {
+			if verified[diskIndex] {
+				// Already validated.
+				return true
+			}
+			// Is this a valid block?
+			isValid := isValidBlock(orderedDisks[diskIndex], volume, path, orderedBlockCheckSums[diskIndex])
+			verified[diskIndex] = isValid
+			return isValid
+		}
+	}()
+
+	// Total bytes written to writer
+	bytesWritten := int64(0)
+
+	// chunkSize is roughly BlockSize/DataBlocks.
+	// chunkSize is calculated such that chunkSize*DataBlocks accommodates BlockSize bytes.
+	// So chunkSize*DataBlocks can be slightly larger than BlockSize if BlockSize is not divisible by
+	// DataBlocks. The extra space will have 0-padding.
+	chunkSize := getEncodedBlockLen(eInfo.BlockSize, eInfo.DataBlocks)
+
+	// Get start and end block, also bytes to be skipped based on the input offset.
+	startBlock, endBlock, bytesToSkip := getBlockInfo(offset, totalLength, eInfo.BlockSize)
+
+	// For each block, read chunk from each disk. If we are able to read all the data disks then we don't
+	// need to read parity disks. If one of the data disk is missing we need to read DataBlocks+1 number
+	// of disks. Once read, we Reconstruct() missing data if needed and write it to the given writer.
+	for block := startBlock; bytesWritten < length; block++ {
+		// Each element of enBlocks holds curChunkSize'd amount of data read from its corresponding disk.
+		enBlocks := make([][]byte, len(orderedDisks))
+
+		// enBlocks data can have 0-padding hence we need to figure the exact number
+		// of bytes we want to read from enBlocks.
+		blockSize := eInfo.BlockSize
+
+		// curChunkSize is chunkSize until end block.
+		curChunkSize := chunkSize
+
+		// We have endBlock, verify if we need to have padding.
+		if block == endBlock && (totalLength%eInfo.BlockSize != 0) {
+			// If this is the last block and size of the block is < BlockSize.
+			curChunkSize = getEncodedBlockLen(totalLength%eInfo.BlockSize, eInfo.DataBlocks)
+
+			// For the last block, the block size can be less than BlockSize.
+			blockSize = totalLength % eInfo.BlockSize
 		}
 
-		// Calculate the current encoded block size.
-		curEncBlockSize := getEncodedBlockLen(curBlockSize, eInfo.DataBlocks)
-		offsetEncOffset := getEncodedBlockLen(startOffset, eInfo.DataBlocks)
+		// Block offset.
+		// NOTE: That for the offset calculation we have to use chunkSize and
+		// not curChunkSize. If we use curChunkSize for offset calculation
+		// then it can result in wrong offset for the last block.
+		blockOffset := block * chunkSize
 
-		// Allocate encoded blocks up to storage disks.
-		enBlocks := make([][]byte, len(disks))
+		// nextIndex - index from which next set of parallel reads
+		// should happen.
+		nextIndex := 0
 
-		// Counter to keep success data blocks.
-		var successDataBlocksCount = 0
-		var noReconstruct bool // Set for no reconstruction.
-
-		// Read from all the disks.
-		for index, disk := range disks {
-			blockIndex := eInfo.Distribution[index] - 1
-			if !isValidBlock(disks, volume, path, toDiskIndex(blockIndex, eInfo.Distribution), blockCheckSums) {
-				continue
-			}
-			if disk == nil {
-				continue
-			}
-			// Initialize shard slice and fill the data from each parts.
-			enBlocks[blockIndex] = make([]byte, curEncBlockSize)
-			// Read the necessary blocks.
-			_, err := disk.ReadFile(volume, path, offsetEncOffset, enBlocks[blockIndex])
+		for {
+			// readDisks - disks from which we need to read in parallel.
+			var readDisks []StorageAPI
+			var err error
+			readDisks, nextIndex, err = getReadDisks(orderedDisks, nextIndex, eInfo.DataBlocks)
 			if err != nil {
-				enBlocks[blockIndex] = nil
+				return bytesWritten, err
 			}
-			// Verify if we have successfully read all the data blocks.
-			if blockIndex < eInfo.DataBlocks && enBlocks[blockIndex] != nil {
-				successDataBlocksCount++
-				// Set when we have all the data blocks and no
-				// reconstruction is needed, so that we can avoid
-				// erasure reconstruction.
-				noReconstruct = successDataBlocksCount == eInfo.DataBlocks
-				if noReconstruct {
-					// Break out we have read all the data blocks.
-					break
-				}
+			parallelRead(volume, path, readDisks, orderedDisks, enBlocks, blockOffset, curChunkSize, bitRotVerify)
+			if isSuccessDecodeBlocks(enBlocks, eInfo.DataBlocks) {
+				// If enough blocks are available to do rs.Reconstruct()
+				break
+			}
+			if nextIndex == len(orderedDisks) {
+				// No more disks to read from.
+				return bytesWritten, errXLReadQuorum
 			}
 		}
 
-		// Check blocks if they are all zero in length, we have corruption return error.
-		if checkBlockSize(enBlocks) == 0 {
-			return nil, errXLDataCorrupt
-		}
-
-		// Verify if reconstruction is needed, proceed with reconstruction.
-		if !noReconstruct {
-			err := decodeData(enBlocks, eInfo.DataBlocks, eInfo.ParityBlocks)
-			if err != nil {
-				return nil, err
+		// If we have all the data blocks no need to decode, continue to write.
+		if !isSuccessDataBlocks(enBlocks, eInfo.DataBlocks) {
+			// Reconstruct the missing data blocks.
+			if err := decodeData(enBlocks, eInfo.DataBlocks, eInfo.ParityBlocks); err != nil {
+				return bytesWritten, err
 			}
 		}
 
-		// Get data blocks from encoded blocks.
-		dataBlocks, err := getDataBlocks(enBlocks, eInfo.DataBlocks, int(curBlockSize))
+		var outSize, outOffset int64
+		// If this is start block, skip unwanted bytes.
+		if block == startBlock {
+			outOffset = bytesToSkip
+		}
+
+		// Total data to be read.
+		outSize = blockSize
+		if length-bytesWritten < blockSize {
+			// We should not send more data than what was requested.
+			outSize = length - bytesWritten
+		}
+
+		// Write data blocks.
+		n, err := writeDataBlocks(writer, enBlocks, eInfo.DataBlocks, outOffset, outSize)
 		if err != nil {
-			return nil, err
+			return bytesWritten, err
 		}
 
-		// Copy data blocks.
-		buffer = append(buffer, dataBlocks...)
-
-		// Negate the 'n' size written to client.
-		totalSizeLeft -= int64(len(dataBlocks))
-
-		// Increase the offset to move forward.
-		startOffset += int64(len(dataBlocks))
-
-		// Relenquish memory.
-		dataBlocks = nil
+		// Update total bytes written.
+		bytesWritten += n
 	}
-	return buffer, nil
+
+	// Success.
+	return bytesWritten, nil
 }
 
 // PartObjectChecksum - returns the checksum for the part name from the checksum slice.
@@ -152,48 +331,37 @@ func metaPartBlockChecksums(disks []StorageAPI, eInfos []erasureInfo, partName s
 	return blockCheckSums
 }
 
-// Takes block index and block distribution to get the disk index.
-func toDiskIndex(blockIdx int, distribution []int) (diskIndex int) {
-	diskIndex = -1
-	// Find out the right disk index for the input block index.
-	for index, blockIndex := range distribution {
-		if blockIndex == blockIdx {
-			diskIndex = index
-		}
-	}
-	return diskIndex
-}
-
 // isValidBlock - calculates the checksum hash for the block and
 // validates if its correct returns true for valid cases, false otherwise.
-func isValidBlock(disks []StorageAPI, volume, path string, diskIndex int, blockCheckSums []checkSumInfo) bool {
-	// Unknown block index requested, treat it as error.
-	if diskIndex == -1 {
-		return false
-	}
-	// Disk is not present, treat entire block to be non existent.
-	if disks[diskIndex] == nil {
+func isValidBlock(disk StorageAPI, volume, path string, blockCheckSum checkSumInfo) (ok bool) {
+	// Disk is not available, not a valid block.
+	if disk == nil {
 		return false
 	}
 	// Read everything for a given block and calculate hash.
-	hashWriter := newHash(blockCheckSums[diskIndex].Algorithm)
-	hashBytes, err := hashSum(disks[diskIndex], volume, path, hashWriter)
+	hashWriter := newHash(blockCheckSum.Algorithm)
+	hashBytes, err := hashSum(disk, volume, path, hashWriter)
 	if err != nil {
+		errorIf(err, "Unable to calculate checksum %s/%s", volume, path)
 		return false
 	}
-	return hex.EncodeToString(hashBytes) == blockCheckSums[diskIndex].Hash
+	return hex.EncodeToString(hashBytes) == blockCheckSum.Hash
 }
 
 // decodeData - decode encoded blocks.
 func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
+	// Initialized reedsolomon.
 	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
 	if err != nil {
 		return err
 	}
+
+	// Reconstruct encoded blocks.
 	err = rs.Reconstruct(enBlocks)
 	if err != nil {
 		return err
 	}
+
 	// Verify reconstructed blocks (parity).
 	ok, err := rs.Verify(enBlocks)
 	if err != nil {
@@ -204,5 +372,7 @@ func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
 		err = errors.New("Verification failed after reconstruction, data likely corrupted.")
 		return err
 	}
+
+	// Success.
 	return nil
 }

erasure-readfile_test.go

@@ -0,0 +1,206 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "testing"
+import "reflect"
+
+// Tests getReadDisks which returns readable disks slice from which we can
+// read parallelly.
+func testGetReadDisks(t *testing.T, xl xlObjects) {
+	d := xl.storageDisks
+	testCases := []struct {
+		index     int          // index argument for getReadDisks
+		argDisks  []StorageAPI // disks argument for getReadDisks
+		retDisks  []StorageAPI // disks return value from getReadDisks
+		nextIndex int          // return value from getReadDisks
+		err       error        // error return value from getReadDisks
+	}{
+		{
+			0,
+			// When all disks are available, should return data disks.
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
+			8,
+			nil,
+		},
+		{
+			0,
+			// If a parity disk is down, should return all data disks.
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], nil, d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
+			8,
+			nil,
+		},
+		{
+			0,
+			// If a data disk is down, should return 7 data and 2 parity.
+			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], nil, nil, nil, nil, nil, nil},
+			10,
+			nil,
+		},
+		{
+			0,
+			// If 7 data disks are down, should return 1 data and 8 parity.
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			16,
+			nil,
+		},
+		{
+			8,
+			// When all disks are available, should return data disks.
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, d[8], d[9], d[10], nil, nil, nil, nil, nil},
+			11,
+			nil,
+		},
+		{
+			11,
+			// When all disks are available, should return data disks.
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, d[11], d[12], nil, nil, nil},
+			13,
+			nil,
+		},
+		{
+			13,
+			// When all disks are available, should return data disks.
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], nil, nil, nil, nil, nil},
+			nil,
+			0,
+			errXLReadQuorum,
+		},
+	}
+
+	for i, test := range testCases {
+		disks, nextIndex, err := getReadDisks(test.argDisks, test.index, xl.dataBlocks)
+		if err != test.err {
+			t.Errorf("test-case %d - expected error : %s, got : %s", i+1, test.err, err)
+			continue
+		}
+		if test.nextIndex != nextIndex {
+			t.Errorf("test-case %d - expected nextIndex: %d, got : %d", i+1, test.nextIndex, nextIndex)
+			continue
+		}
+		if reflect.DeepEqual(test.retDisks, disks) == false {
+			t.Errorf("test-case %d : incorrect disks returned. %v", i+1, disks)
+			continue
+		}
+	}
+}
+
+// Test getOrderedDisks which returns ordered slice of disks from their
+// actual distribution.
+func testGetOrderedDisks(t *testing.T, xl xlObjects) {
+	disks := xl.storageDisks
+	blockCheckSums := make([]checkSumInfo, len(disks))
+	distribution := []int{16, 14, 12, 10, 8, 6, 4, 2, 1, 3, 5, 7, 9, 11, 13, 15}
+	orderedDisks, _ := getOrderedDisks(distribution, disks, blockCheckSums)
+	// From the "distribution" above you can notice that:
+	// 1st data block is in the 9th disk (i.e distribution index 8)
+	// 2nd data block is in the 8th disk (i.e distribution index 7) and so on.
+	if orderedDisks[0] != disks[8] ||
+		orderedDisks[1] != disks[7] ||
+		orderedDisks[2] != disks[9] ||
+		orderedDisks[3] != disks[6] ||
+		orderedDisks[4] != disks[10] ||
+		orderedDisks[5] != disks[5] ||
+		orderedDisks[6] != disks[11] ||
+		orderedDisks[7] != disks[4] ||
+		orderedDisks[8] != disks[12] ||
+		orderedDisks[9] != disks[3] ||
+		orderedDisks[10] != disks[13] ||
+		orderedDisks[11] != disks[2] ||
+		orderedDisks[12] != disks[14] ||
+		orderedDisks[13] != disks[1] ||
+		orderedDisks[14] != disks[15] ||
+		orderedDisks[15] != disks[0] {
+		t.Errorf("getOrderedDisks returned incorrect order.")
+	}
+}
+
+// Test for isSuccessDataBlocks and isSuccessDecodeBlocks.
+func TestIsSuccessBlocks(t *testing.T) {
+	dataBlocks := 8
+	testCases := []struct {
+		enBlocks      [][]byte // data and parity blocks.
+		successData   bool     // expected return value of isSuccessDataBlocks()
+		successDecode bool     // expected return value of isSuccessDecodeBlocks()
+	}{
+		{
+			// When all data and partity blocks are available.
+			[][]byte{
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			true,
+			true,
+		},
+		{
+			// When one data block is not available.
+			[][]byte{
+				nil, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// When one data and all parity are available, enough for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// Not enough disks for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			false,
+		},
+	}
+
+	for i, test := range testCases {
+		got := isSuccessDataBlocks(test.enBlocks, dataBlocks)
+		if test.successData != got {
+			t.Errorf("test-case %d : expected %v got %v", i+1, test.successData, got)
+		}
+		got = isSuccessDecodeBlocks(test.enBlocks, dataBlocks)
+		if test.successDecode != got {
+			t.Errorf("test-case %d : expected %v got %v", i+1, test.successDecode, got)
+		}
+	}
+}
+
+// Wrapper function for testGetReadDisks, testGetOrderedDisks.
+func TestErasureReadFile(t *testing.T) {
+	objLayer, dirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeRoots(dirs)
+	xl := objLayer.(xlObjects)
+	testGetReadDisks(t, xl)
+	testGetOrderedDisks(t, xl)
+}

erasure-utils.go

@@ -17,18 +17,20 @@
 package main
 
 import (
-	"crypto/sha512"
+	"bytes"
+	"errors"
 	"hash"
 	"io"
 
 	"github.com/klauspost/reedsolomon"
+	"github.com/minio/blake2b-simd"
 )
 
 // newHashWriters - inititialize a slice of hashes for the disk count.
 func newHashWriters(diskCount int) []hash.Hash {
 	hashWriters := make([]hash.Hash, diskCount)
 	for index := range hashWriters {
-		hashWriters[index] = newHash("sha512")
+		hashWriters[index] = newHash("blake2b")
 	}
 	return hashWriters
 }
@@ -36,68 +38,111 @@ func newHashWriters(diskCount int) []hash.Hash {
 // newHash - gives you a newly allocated hash depending on the input algorithm.
 func newHash(algo string) hash.Hash {
 	switch algo {
-	case "sha512":
-		return sha512.New()
+	case "blake2b":
+		return blake2b.New512()
 	// Add new hashes here.
 	default:
-		return sha512.New()
+		// Default to blake2b.
+		return blake2b.New512()
 	}
 }
 
+// hashSum calculates the hash of the entire path and returns.
 func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
-	startOffset := int64(0)
-	// Read until io.EOF.
-	for {
-		buf := make([]byte, blockSizeV1)
-		n, err := disk.ReadFile(volume, path, startOffset, buf)
-		if err == io.EOF {
-			break
-		}
-		if err != nil && err != io.EOF {
-			return nil, err
-		}
-		writer.Write(buf[:n])
-		startOffset += n
+	// Allocate staging buffer of 128KiB for copyBuffer.
+	buf := make([]byte, readSizeV1)
+
+	// Copy entire buffer to writer.
+	if err := copyBuffer(writer, disk, volume, path, buf); err != nil {
+		return nil, err
 	}
+
+	// Return the final hash sum.
 	return writer.Sum(nil), nil
 }
 
-// getDataBlocks - fetches the data block only part of the input encoded blocks.
-func getDataBlocks(enBlocks [][]byte, dataBlocks int, curBlockSize int) (data []byte, err error) {
-	if len(enBlocks) < dataBlocks {
-		return nil, reedsolomon.ErrTooFewShards
-	}
+// getDataBlockLen - get length of data blocks from encoded blocks.
+func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
 	size := 0
-	blocks := enBlocks[:dataBlocks]
-	for _, block := range blocks {
+	// Figure out the data block length.
+	for _, block := range enBlocks[:dataBlocks] {
 		size += len(block)
 	}
-	if size < curBlockSize {
-		return nil, reedsolomon.ErrShortData
-	}
-
-	write := curBlockSize
-	for _, block := range blocks {
-		if write < len(block) {
-			data = append(data, block[:write]...)
-			return data, nil
-		}
-		data = append(data, block...)
-		write -= len(block)
-	}
-	return data, nil
+	return size
 }
 
-// checkBlockSize return the size of a single block.
-// The first non-zero size is returned,
-// or 0 if all blocks are size 0.
-func checkBlockSize(blocks [][]byte) int {
-	for _, block := range blocks {
-		if len(block) != 0 {
-			return len(block)
-		}
+// Writes all the data blocks from encoded blocks until requested
+// outSize length. Provides a way to skip bytes until the offset.
+func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, outOffset int64, outSize int64) (int64, error) {
+	// Offset and out size cannot be negative.
+	if outOffset < 0 || outSize < 0 {
+		return 0, errUnexpected
 	}
-	return 0
+
+	// Do we have enough blocks?
+	if len(enBlocks) < dataBlocks {
+		return 0, reedsolomon.ErrTooFewShards
+	}
+
+	// Do we have enough data?
+	if int64(getDataBlockLen(enBlocks, dataBlocks)) < outSize {
+		return 0, reedsolomon.ErrShortData
+	}
+
+	// Counter to decrement total left to write.
+	write := outSize
+
+	// Counter to increment total written.
+	totalWritten := int64(0)
+
+	// Write all data blocks to dst.
+	for _, block := range enBlocks[:dataBlocks] {
+		// Skip blocks until we have reached our offset.
+		if outOffset >= int64(len(block)) {
+			// Decrement offset.
+			outOffset -= int64(len(block))
+			continue
+		} else {
+			// Skip until offset.
+			block = block[outOffset:]
+
+			// Reset the offset for next iteration to read everything
+			// from subsequent blocks.
+			outOffset = 0
+		}
+		// We have written all the blocks, write the last remaining block.
+		if write < int64(len(block)) {
+			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
+			if err != nil {
+				return 0, err
+			}
+			totalWritten += n
+			break
+		}
+		// Copy the block.
+		n, err := io.Copy(dst, bytes.NewReader(block))
+		if err != nil {
+			return 0, err
+		}
+
+		// Decrement output size.
+		write -= n
+
+		// Increment written.
+		totalWritten += n
+	}
+
+	// Success.
+	return totalWritten, nil
+}
+
+// getBlockInfo - find start/end block and bytes to skip for given offset, length and block size.
+func getBlockInfo(offset, length, blockSize int64) (startBlock, endBlock, bytesToSkip int64) {
+	// Calculate start block for given offset and how many bytes to skip to get the offset.
+	startBlock = offset / blockSize
+	bytesToSkip = offset % blockSize
+	endBlock = length / blockSize
+	return
 }
 
 // calculate the blockSize based on input length and total number of
@@ -106,3 +151,95 @@ func getEncodedBlockLen(inputLen int64, dataBlocks int) (curEncBlockSize int64)
 	curEncBlockSize = (inputLen + int64(dataBlocks) - 1) / int64(dataBlocks)
 	return curEncBlockSize
 }
+
+// copyN - copies from disk, volume, path to input writer until length
+// is reached at volume, path or an error occurs. A success copyN returns
+// err == nil, not err == EOF. Additionally offset can be provided to start
+// the read at. copyN returns io.EOF if there aren't enough data to be read.
+func copyN(writer io.Writer, disk StorageAPI, volume string, path string, offset int64, length int64) (err error) {
+	// Use staging buffer to read up to length.
+	bufSize := int64(readSizeV1)
+	if length > 0 && bufSize > length {
+		bufSize = length
+	}
+	buf := make([]byte, int(bufSize))
+
+	// Read into writer until length.
+	for length > 0 {
+		curLength := bufSize
+		if length < bufSize {
+			curLength = length
+		}
+		nr, er := disk.ReadFile(volume, path, offset, buf[:curLength])
+		if nr > 0 {
+			nw, ew := writer.Write(buf[0:nr])
+			if nw > 0 {
+				// Decrement the length.
+				length -= int64(nw)
+
+				// Progress the offset.
+				offset += int64(nw)
+			}
+			if ew != nil {
+				err = ew
+				break
+			}
+			if nr != int64(nw) {
+				err = io.ErrShortWrite
+				break
+			}
+		}
+		if er == io.EOF || er == io.ErrUnexpectedEOF {
+			break
+		}
+		if er != nil {
+			err = er
+			break
+		}
+	}
+
+	// Success.
+	return err
+}
+
+// copyBuffer - copies from disk, volume, path to input writer until either EOF
+// is reached at volume, path or an error occurs. A success copyBuffer returns
+// err == nil, not err == EOF. Because copyBuffer is defined to read from path
+// until EOF. It does not treat an EOF from ReadFile an error to be reported.
+// Additionally copyBuffer stages through the provided buffer; otherwise if it
+// has zero length, returns error.
+func copyBuffer(writer io.Writer, disk StorageAPI, volume string, path string, buf []byte) error {
+	// Error condition of zero length buffer.
+	if buf != nil && len(buf) == 0 {
+		return errors.New("empty buffer in readBuffer")
+	}
+
+	// Starting offset for Reading the file.
+	startOffset := int64(0)
+
+	// Read until io.EOF.
+	for {
+		n, err := disk.ReadFile(volume, path, startOffset, buf)
+		if n > 0 {
+			var m int
+			m, err = writer.Write(buf[:n])
+			if err != nil {
+				return err
+			}
+			if int64(m) != n {
+				return io.ErrShortWrite
+			}
+		}
+		if err != nil {
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				break
+			}
+			return err
+		}
+		// Progress the offset.
+		startOffset += n
+	}
+
+	// Success.
+	return nil
+}

erasure-waitcloser.go

@@ -1,74 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"io"
-	"sync"
-)
-
-// waitCloser implements a Closer that blocks until released, this
-// prevents the writer closing one end of a pipe while making sure
-// that all data is written and committed to disk on the end.
-// Additionally this also implements Write().
-type waitCloser struct {
-	wg     *sync.WaitGroup // Waitgroup for atomicity.
-	writer io.WriteCloser  // Embedded writer.
-}
-
-// Write to the underlying writer.
-func (b *waitCloser) Write(data []byte) (int, error) {
-	return b.writer.Write(data)
-}
-
-// Close blocks until another goroutine calls Release(error). Returns
-// error code if either writer fails or Release is called with an error.
-func (b *waitCloser) Close() error {
-	err := b.writer.Close()
-	b.wg.Wait()
-	return err
-}
-
-// CloseWithError closes the writer; subsequent read to the read
-// half of the pipe will return the error err.
-func (b *waitCloser) CloseWithError(err error) error {
-	w, ok := b.writer.(*io.PipeWriter)
-	if ok {
-		return w.CloseWithError(err)
-	}
-	return err
-}
-
-// release the Close, causing it to unblock. Only call this
-// once. Calling it multiple times results in a panic.
-func (b *waitCloser) release() {
-	b.wg.Done()
-	return
-}
-
-// newWaitCloser creates a new write closer that must be
-// released by the read consumer.
-func newWaitCloser(writer io.WriteCloser) *waitCloser {
-	// Wait group for the go-routine.
-	wg := &sync.WaitGroup{}
-	// Add to the wait group to wait for.
-	wg.Add(1)
-	return &waitCloser{
-		wg:     wg,
-		writer: writer,
-	}
-}

erasure_test.go

@@ -0,0 +1,185 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"testing"
+)
+
+// mustEncodeData - encodes data slice and provides encoded 2 dimensional slice.
+func mustEncodeData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData, err := encodeData(data, dataBlocks, parityBlocks)
+	if err != nil {
+		// Upon failure panic this function.
+		panic(err)
+	}
+	return encodedData
+}
+
+// Generates good encoded data with one parity block and data block missing.
+func getGoodEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	encodedData[3] = nil
+	encodedData[1] = nil
+	return encodedData
+}
+
+// Generates bad encoded data with one parity block and data block with garbage data.
+func getBadEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	encodedData[3] = []byte("garbage")
+	encodedData[1] = []byte("garbage")
+	return encodedData
+}
+
+// Generates encoded data with all data blocks missing.
+func getMissingData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	for i := 0; i < dataBlocks+1; i++ {
+		encodedData[i] = nil
+	}
+	return encodedData
+}
+
+// Generates encoded data with less number of blocks than expected data blocks.
+func getInsufficientData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	// Return half of the data.
+	return encodedData[:dataBlocks/2]
+}
+
+// Represents erasure encoding matrix dataBlocks and paritBlocks.
+type encodingMatrix struct {
+	dataBlocks   int
+	parityBlocks int
+}
+
+// List of encoding matrices the tests will run on.
+var encodingMatrices = []encodingMatrix{
+	{3, 3}, // 3 data, 3 parity blocks.
+	{4, 4}, // 4 data, 4 parity blocks.
+	{5, 5}, // 5 data, 5 parity blocks.
+	{6, 6}, // 6 data, 6 parity blocks.
+	{7, 7}, // 7 data, 7 parity blocks.
+	{8, 8}, // 8 data, 8 parity blocks.
+}
+
+// TestErasureEncode checks for encoding for different data sets.
+func TestErasureEncode(t *testing.T) {
+	// Collection of cases for encode cases.
+	testEncodeCases := []struct {
+		inputData  []byte
+		shouldPass bool
+	}{
+		// Regular data encoded.
+		{
+			inputData:  []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."),
+			shouldPass: true,
+		},
+		// Empty data errors out.
+		{
+			inputData:  []byte(""),
+			shouldPass: false,
+		},
+		// Single byte encoded.
+		{
+			inputData:  []byte("1"),
+			shouldPass: true,
+		},
+	}
+
+	// Test encode cases.
+	for i, testCase := range testEncodeCases {
+		for _, encodingMatrix := range encodingMatrices {
+			_, err := encodeData(testCase.inputData, encodingMatrix.dataBlocks, encodingMatrix.parityBlocks)
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: Expected to pass by failed instead with %s, for matrix %v", i+1, err, encodingMatrix)
+			}
+		}
+	}
+}
+
+// Tests erasure decoding functionality for various types of inputs.
+func TestErasureDecode(t *testing.T) {
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+
+	// List of decoding cases
+	// - validates bad encoded data.
+	// - validates good encoded data.
+	// - validates insufficient data.
+	testDecodeCases := []struct {
+		enFn       func([]byte, int, int) [][]byte
+		shouldPass bool
+	}{
+		// Generates bad encoded data.
+		{
+			enFn:       getBadEncodedData,
+			shouldPass: false,
+		},
+		// Generates good encoded data.
+		{
+			enFn:       getGoodEncodedData,
+			shouldPass: true,
+		},
+		// Generates missing data.
+		{
+			enFn:       getMissingData,
+			shouldPass: false,
+		},
+		// Generates short data.
+		{
+			enFn:       getInsufficientData,
+			shouldPass: false,
+		},
+	}
+
+	// Validates all decode tests.
+	for i, testCase := range testDecodeCases {
+		for _, encodingMatrix := range encodingMatrices {
+
+			// Encoding matrix.
+			dataBlocks := encodingMatrix.dataBlocks
+			parityBlocks := encodingMatrix.parityBlocks
+
+			// Data block size.
+			blockSize := len(data)
+
+			// Generates encoded data based on type of testCase function.
+			encodedData := testCase.enFn(data, dataBlocks, parityBlocks)
+
+			// Decodes the data.
+			err := decodeData(encodedData, dataBlocks, parityBlocks)
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
+			}
+
+			// Proceed to extract the data blocks.
+			decodedDataWriter := new(bytes.Buffer)
+			_, err = writeDataBlocks(decodedDataWriter, encodedData, dataBlocks, 0, int64(blockSize))
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
+			}
+
+			// Validate if decoded data is what we expected.
+			if bytes.Equal(decodedDataWriter.Bytes(), data) != testCase.shouldPass {
+				err := errUnexpected
+				t.Errorf("Test %d: Expected to pass by failed instead %s", i+1, err)
+			}
+		}
+	}
+}

format-config-v1.go

@@ -22,8 +22,6 @@ import (
 	"fmt"
 	"strings"
 	"sync"
-
-	"github.com/skyrings/skyring-common/tools/uuid"
 )
 
 // fsFormat - structure holding 'fs' format.
@@ -107,31 +105,41 @@ else // No healing at this point forward, some disks are offline or dead.
 fi
 */
 
+// error returned when some disks are found to be unformatted.
 var errSomeDiskUnformatted = errors.New("some disks are found to be unformatted")
+
+// error returned when some disks are offline.
 var errSomeDiskOffline = errors.New("some disks are offline")
 
+// errDiskOrderMismatch - returned when disk UUID is not in consistent JBOD order.
+var errDiskOrderMismatch = errors.New("disk order mismatch")
+
 // Returns error slice into understandable errors.
-func reduceFormatErrs(errs []error, diskCount int) error {
+func reduceFormatErrs(errs []error, diskCount int) (err error) {
 	var errUnformattedDiskCount = 0
 	var errDiskNotFoundCount = 0
-	for _, err := range errs {
-		if err == errUnformattedDisk {
+	for _, dErr := range errs {
+		if dErr == errUnformattedDisk {
 			errUnformattedDiskCount++
-		} else if err == errDiskNotFound {
+		} else if dErr == errDiskNotFound {
 			errDiskNotFoundCount++
 		}
 	}
-	// Returns errUnformattedDisk if all disks report unFormattedDisk.
-	if errUnformattedDiskCount == diskCount {
+	// Unformatted disks found, we need to figure out if any disks are offline.
+	if errUnformattedDiskCount > 0 {
+		// Returns errUnformattedDisk if all disks report unFormattedDisk.
+		if errUnformattedDiskCount < diskCount {
+			if errDiskNotFoundCount > 0 {
+				// Only some disks are fresh but some disks are offline as well.
+				return errSomeDiskOffline
+			}
+			// Some disks are fresh disks an unformatted, not disks are offline.
+			return errSomeDiskUnformatted
+		}
+		// All disks returned unformatted, all disks must be fresh.
 		return errUnformattedDisk
-	} else if errUnformattedDiskCount < diskCount && errDiskNotFoundCount == 0 {
-		// Only some disks return unFormattedDisk and all disks are online.
-		return errSomeDiskUnformatted
-	} else if errUnformattedDiskCount < diskCount && errDiskNotFoundCount > 0 {
-		// Only some disks return unFormattedDisk and some disks are
-		// offline as well.
-		return errSomeDiskOffline
 	}
+	// No unformatted disks found no need to handle disk not found case, return success here.
 	return nil
 }
 
@@ -148,6 +156,10 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
 
 	// Make a volume entry on all underlying storage disks.
 	for index, disk := range bootstrapDisks {
+		if disk == nil {
+			sErrs[index] = errDiskNotFound
+			continue
+		}
 		wg.Add(1)
 		// Make a volume inside a go-routine.
 		go func(index int, disk StorageAPI) {
@@ -224,33 +236,59 @@ func genericFormatCheck(formatConfigs []*formatConfigV1, sErrs []error) (err err
 	return nil
 }
 
+// isSavedUUIDInOrder - validates if disk uuid is present and valid in all
+// available format config JBOD. This function also validates if the disk UUID
+// is always available on all JBOD under the same order.
+func isSavedUUIDInOrder(uuid string, formatConfigs []*formatConfigV1) bool {
+	var orderIndexes []int
+	// Validate each for format.json for relevant uuid.
+	for _, formatConfig := range formatConfigs {
+		if formatConfig == nil {
+			continue
+		}
+		// Validate if UUID is present in JBOD.
+		uuidIndex := findDiskIndex(uuid, formatConfig.XL.JBOD)
+		if uuidIndex == -1 {
+			// UUID not found.
+			errorIf(errDiskNotFound, "Disk %s not found in JBOD list", uuid)
+			return false
+		}
+		// Save the position of UUID present in JBOD.
+		orderIndexes = append(orderIndexes, uuidIndex+1)
+	}
+	// Once uuid is found, verify if the uuid
+	// present in same order across all format configs.
+	prevOrderIndex := orderIndexes[0]
+	for _, orderIndex := range orderIndexes {
+		if prevOrderIndex != orderIndex {
+			errorIf(errDiskOrderMismatch, "Disk %s is in wrong order wanted %d, saw %d ", uuid, prevOrderIndex, orderIndex)
+			return false
+		}
+	}
+	// Returns success, when we have verified if uuid
+	// is consistent and valid across all format configs.
+	return true
+}
+
 // checkDisksConsistency - checks if all disks are consistent with all JBOD entries on all disks.
 func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
 	var disks = make([]string, len(formatConfigs))
-	var disksFound = make(map[string]bool)
 	// Collect currently available disk uuids.
 	for index, formatConfig := range formatConfigs {
 		if formatConfig == nil {
+			disks[index] = ""
 			continue
 		}
 		disks[index] = formatConfig.XL.Disk
 	}
 	// Validate collected uuids and verify JBOD.
-	for index, uuid := range disks {
+	for _, uuid := range disks {
 		if uuid == "" {
 			continue
 		}
-		var formatConfig = formatConfigs[index]
-		for _, savedUUID := range formatConfig.XL.JBOD {
-			if savedUUID == uuid {
-				disksFound[uuid] = true
-			}
-		}
-	}
-	// Check if all disks are found.
-	for _, value := range disksFound {
-		if !value {
-			return errors.New("Some disks not found in JBOD.")
+		// Is uuid present on all JBOD ?.
+		if !isSavedUUIDInOrder(uuid, formatConfigs) {
+			return fmt.Errorf("%s disk not found in JBOD", uuid)
 		}
 	}
 	return nil
@@ -258,16 +296,15 @@ func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
 
 // checkJBODConsistency - validate xl jbod order if they are consistent.
 func checkJBODConsistency(formatConfigs []*formatConfigV1) error {
-	var firstJBOD []string
+	var jbodStr string
 	// Extract first valid JBOD.
 	for _, format := range formatConfigs {
 		if format == nil {
 			continue
 		}
-		firstJBOD = format.XL.JBOD
+		jbodStr = strings.Join(format.XL.JBOD, ".")
 		break
 	}
-	jbodStr := strings.Join(firstJBOD, ".")
 	for _, format := range formatConfigs {
 		if format == nil {
 			continue
@@ -280,7 +317,8 @@ func checkJBODConsistency(formatConfigs []*formatConfigV1) error {
 	return nil
 }
 
-func findIndex(disk string, jbod []string) int {
+// findDiskIndex returns position of disk in JBOD.
+func findDiskIndex(disk string, jbod []string) int {
 	for index, uuid := range jbod {
 		if uuid == disk {
 			return index
@@ -305,7 +343,7 @@ func reorderDisks(bootstrapDisks []StorageAPI, formatConfigs []*formatConfigV1)
 		if format == nil {
 			continue
 		}
-		jIndex := findIndex(format.XL.Disk, savedJBOD)
+		jIndex := findDiskIndex(format.XL.Disk, savedJBOD)
 		if jIndex == -1 {
 			return nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found")
 		}
@@ -314,10 +352,9 @@ func reorderDisks(bootstrapDisks []StorageAPI, formatConfigs []*formatConfigV1)
 	return newDisks, nil
 }
 
-// loadFormat - load format from disk.
+// loadFormat - loads format.json from disk.
 func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
-	var buffer []byte
-	buffer, err = readAll(disk, minioMetaBucket, formatConfigFile)
+	buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
 	if err != nil {
 		// 'file not found' and 'volume not found' as
 		// same. 'volume not found' usually means its a fresh disk.
@@ -336,31 +373,59 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
 		}
 		return nil, err
 	}
+
+	// Try to decode format json into formatConfigV1 struct.
 	format = &formatConfigV1{}
-	err = json.Unmarshal(buffer, format)
-	if err != nil {
+	if err = json.Unmarshal(buf, format); err != nil {
 		return nil, err
 	}
+
+	// Success.
 	return format, nil
 }
 
+// isFormatNotFound - returns true if all `format.json` are not
+// found on all disks.
+func isFormatNotFound(formats []*formatConfigV1) bool {
+	for _, format := range formats {
+		// One of the `format.json` is found.
+		if format != nil {
+			return false
+		}
+	}
+	// All format.json missing, success.
+	return true
+}
+
+// isFormatFound - returns true if all input formats are found on
+// all disks.
+func isFormatFound(formats []*formatConfigV1) bool {
+	for _, format := range formats {
+		// One of `format.json` is not found.
+		if format == nil {
+			return false
+		}
+	}
+	// All format.json present, success.
+	return true
+}
+
 // Heals any missing format.json on the drives. Returns error only for unexpected errors
 // as regular errors can be ignored since there might be enough quorum to be operational.
-func healFormatXL(bootstrapDisks []StorageAPI) error {
-	needHeal := make([]bool, len(bootstrapDisks)) // Slice indicating which drives needs healing.
-
-	formatConfigs := make([]*formatConfigV1, len(bootstrapDisks))
+func healFormatXL(storageDisks []StorageAPI) error {
+	formatConfigs := make([]*formatConfigV1, len(storageDisks))
 	var referenceConfig *formatConfigV1
-	successCount := 0        // Tracks if we have successfully loaded all `format.json` from all disks.
-	formatNotFoundCount := 0 // Tracks if we `format.json` is not found on all disks.
 	// Loads `format.json` from all disks.
-	for index, disk := range bootstrapDisks {
+	for index, disk := range storageDisks {
+		// Disk not found or ignored is a valid case.
+		if disk == nil {
+			// Proceed without healing.
+			return nil
+		}
 		formatXL, err := loadFormat(disk)
 		if err != nil {
 			if err == errUnformattedDisk {
 				// format.json is missing, should be healed.
-				needHeal[index] = true
-				formatNotFoundCount++
 				continue
 			} else if err == errDiskNotFound { // Is a valid case we
 				// can proceed without healing.
@@ -370,16 +435,16 @@ func healFormatXL(bootstrapDisks []StorageAPI) error {
 			return err
 		} // Success.
 		formatConfigs[index] = formatXL
-		successCount++
 	}
 	// All `format.json` has been read successfully, previously completed.
-	if successCount == len(bootstrapDisks) {
+	if isFormatFound(formatConfigs) {
 		// Return success.
 		return nil
 	}
+
 	// All disks are fresh, format.json will be written by initFormatXL()
-	if formatNotFoundCount == len(bootstrapDisks) {
-		return initFormatXL(bootstrapDisks)
+	if isFormatNotFound(formatConfigs) {
+		return initFormatXL(storageDisks)
 	}
 	// Validate format configs for consistency in JBOD and disks.
 	if err := checkFormatXL(formatConfigs); err != nil {
@@ -397,68 +462,40 @@ func healFormatXL(bootstrapDisks []StorageAPI) error {
 		}
 	}
 
-	uuidUsage := make([]struct {
-		uuid  string // Disk uuid
-		inUse bool   // indicates if the uuid is used by
-		// any disk
-	}, len(bootstrapDisks))
+	// Collect new format configs.
+	var newFormatConfigs = make([]*formatConfigV1, len(storageDisks))
 
-	// Returns any unused drive UUID.
-	getUnusedUUID := func() string {
-		for index := range uuidUsage {
-			if !uuidUsage[index].inUse {
-				uuidUsage[index].inUse = true
-				return uuidUsage[index].uuid
-			}
-		}
-		return ""
-	}
+	// Collect new JBOD.
+	newJBOD := referenceConfig.XL.JBOD
 
-	// From reference config update UUID's not be in use.
-	for index, diskUUID := range referenceConfig.XL.JBOD {
-		uuidUsage[index].uuid = diskUUID
-		uuidUsage[index].inUse = false
-	}
-
-	// For all config formats validate if they are in use and
-	// update the uuidUsage values.
-	for _, config := range formatConfigs {
-		if config == nil {
-			continue
-		}
-		for index := range uuidUsage {
-			if config.XL.Disk == uuidUsage[index].uuid {
-				uuidUsage[index].inUse = true
-				break
-			}
-		}
-	}
 	// This section heals the format.json and updates the fresh disks
 	// by apply a new UUID for all the fresh disks.
-	for index, heal := range needHeal {
-		if !heal {
+	for index, format := range formatConfigs {
+		if format == nil {
+			newJBOD[index] = getUUID()
+		}
+	}
+	// Collect new format configs that need to be written.
+	for index, format := range formatConfigs {
+		if format == nil {
+			config := &formatConfigV1{
+				Version: referenceConfig.Version,
+				Format:  referenceConfig.Format,
+				XL: &xlFormat{
+					Version: referenceConfig.XL.Version,
+					Disk:    newJBOD[index],
+					JBOD:    newJBOD,
+				},
+			}
+			newFormatConfigs[index] = config
 			continue
 		}
-		config := &formatConfigV1{}
-		*config = *referenceConfig
-		config.XL.Disk = getUnusedUUID()
-		if config.XL.Disk == "" {
-			// getUnusedUUID() should have
-			// returned an unused uuid, it
-			// is an unexpected error.
-			return errUnexpected
-		}
-
-		formatBytes, err := json.Marshal(config)
-		if err != nil {
-			return err
-		}
-		// Fresh disk without format.json
-		_, _ = bootstrapDisks[index].AppendFile(minioMetaBucket, formatConfigFile, formatBytes)
-		// Ignore any error from AppendFile() as
-		// quorum might still be there to be operational.
+		newFormatConfigs[index] = format
+		newFormatConfigs[index].XL.JBOD = newJBOD
+		newFormatConfigs[index].XL.Disk = newJBOD[index]
 	}
-	return nil
+	// Save new `format.json` across all disks.
+	return saveFormatXL(storageDisks, newFormatConfigs)
 }
 
 // loadFormatXL - loads XL `format.json` and returns back properly
@@ -470,6 +507,10 @@ func loadFormatXL(bootstrapDisks []StorageAPI) (disks []StorageAPI, err error) {
 
 	// Try to load `format.json` bootstrap disks.
 	for index, disk := range bootstrapDisks {
+		if disk == nil {
+			diskNotFoundCount++
+			continue
+		}
 		var formatXL *formatConfigV1
 		formatXL, err = loadFormat(disk)
 		if err != nil {
@@ -486,16 +527,13 @@ func loadFormatXL(bootstrapDisks []StorageAPI) (disks []StorageAPI, err error) {
 		formatConfigs[index] = formatXL
 	}
 
-	// If all disks indicate that 'format.json' is not available
-	// return 'errUnformattedDisk'.
-	if unformattedDisksFoundCnt == len(bootstrapDisks) {
+	// If all disks indicate that 'format.json' is not available return 'errUnformattedDisk'.
+	if unformattedDisksFoundCnt > len(bootstrapDisks)-(len(bootstrapDisks)/2+1) {
 		return nil, errUnformattedDisk
 	} else if diskNotFoundCount == len(bootstrapDisks) {
 		return nil, errDiskNotFound
 	} else if diskNotFoundCount > len(bootstrapDisks)-(len(bootstrapDisks)/2+1) {
 		return nil, errXLReadQuorum
-	} else if unformattedDisksFoundCnt > len(bootstrapDisks)-(len(bootstrapDisks)/2+1) {
-		return nil, errXLReadQuorum
 	}
 
 	// Validate the format configs read are correct.
@@ -532,57 +570,91 @@ func checkFormatXL(formatConfigs []*formatConfigV1) error {
 	return checkDisksConsistency(formatConfigs)
 }
 
-// initFormatXL - save XL format configuration on all disks.
-func initFormatXL(storageDisks []StorageAPI) (err error) {
-	var (
-		jbod             = make([]string, len(storageDisks))
-		formats          = make([]*formatConfigV1, len(storageDisks))
-		saveFormatErrCnt = 0
-	)
+// saveFormatXL - populates `format.json` on disks in its order.
+func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error {
+	var errs = make([]error, len(storageDisks))
+	var wg = &sync.WaitGroup{}
+	// Write `format.json` to all disks.
 	for index, disk := range storageDisks {
-		if err = disk.MakeVol(minioMetaBucket); err != nil {
-			if err != errVolumeExists {
-				saveFormatErrCnt++
-				// Check for write quorum.
-				if saveFormatErrCnt <= len(storageDisks)-(len(storageDisks)/2+3) {
-					continue
-				}
-				return errXLWriteQuorum
-			}
+		if disk == nil {
+			continue
 		}
-		var u *uuid.UUID
-		u, err = uuid.New()
-		if err != nil {
-			saveFormatErrCnt++
-			// Check for write quorum.
-			if saveFormatErrCnt <= len(storageDisks)-(len(storageDisks)/2+3) {
-				continue
+		wg.Add(1)
+		go func(index int, disk StorageAPI, format *formatConfigV1) {
+			defer wg.Done()
+
+			// Marshal and write to disk.
+			formatBytes, err := json.Marshal(format)
+			if err != nil {
+				errs[index] = err
+				return
 			}
+
+			// Purge any existing temporary file, okay to ignore errors here.
+			disk.DeleteFile(minioMetaBucket, formatConfigFileTmp)
+
+			// Append file `format.json.tmp`.
+			if err = disk.AppendFile(minioMetaBucket, formatConfigFileTmp, formatBytes); err != nil {
+				errs[index] = err
+				return
+			}
+			// Rename file `format.json.tmp` --> `format.json`.
+			if err = disk.RenameFile(minioMetaBucket, formatConfigFileTmp, minioMetaBucket, formatConfigFile); err != nil {
+				errs[index] = err
+				return
+			}
+		}(index, disk, formats[index])
+	}
+
+	// Wait for the routines to finish.
+	wg.Wait()
+
+	// Validate if we encountered any errors, return quickly.
+	for _, err := range errs {
+		if err != nil {
+			// Failure.
 			return err
 		}
+	}
+
+	// Success.
+	return nil
+}
+
+// initFormatXL - save XL format configuration on all disks.
+func initFormatXL(storageDisks []StorageAPI) (err error) {
+	// Initialize jbods.
+	var jbod = make([]string, len(storageDisks))
+
+	// Initialize formats.
+	var formats = make([]*formatConfigV1, len(storageDisks))
+
+	// Initialize `format.json`.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		// Allocate format config.
 		formats[index] = &formatConfigV1{
 			Version: "1",
 			Format:  "xl",
 			XL: &xlFormat{
 				Version: "1",
-				Disk:    u.String(),
+				Disk:    getUUID(),
 			},
 		}
 		jbod[index] = formats[index].XL.Disk
 	}
+
+	// Update the jbod entries.
 	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		// Save jbod.
 		formats[index].XL.JBOD = jbod
-		formatBytes, err := json.Marshal(formats[index])
-		if err != nil {
-			return err
-		}
-		n, err := disk.AppendFile(minioMetaBucket, formatConfigFile, formatBytes)
-		if err != nil {
-			return err
-		}
-		if n != int64(len(formatBytes)) {
-			return errUnexpected
-		}
 	}
-	return nil
+
+	// Save formats `format.json` across all disks.
+	return saveFormatXL(storageDisks, formats)
 }

format-config-v1_test.go

@@ -0,0 +1,235 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "testing"
+
+// generates a valid format.json for XL backend.
+func genFormatXLValid() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidVersion() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Corrupt version numbers.
+	formatConfigs[0].Version = "2"
+	formatConfigs[3].Version = "-1"
+	return formatConfigs
+}
+
+// generates a invalid format.json JBOD for XL backend.
+func genFormatXLInvalidJBOD() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	// Corrupt JBOD entries on disk 6 and disk 8.
+	formatConfigs[5].XL.JBOD = jbod
+	formatConfigs[7].XL.JBOD = jbod
+	return formatConfigs
+}
+
+// generates a invalid format.json Disk UUID for XL backend.
+func genFormatXLInvalidDisks() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Make disk 5 and disk 8 have inconsistent disk uuid's.
+	formatConfigs[4].XL.Disk = getUUID()
+	formatConfigs[7].XL.Disk = getUUID()
+	return formatConfigs
+}
+
+// generates a invalid format.json Disk UUID in wrong order for XL backend.
+func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Re order jbod for failure case.
+	var jbod1 = make([]string, 8)
+	for i, j := range jbod {
+		jbod1[i] = j
+	}
+	jbod1[1], jbod1[2] = jbod[2], jbod[1]
+	formatConfigs[2].XL.JBOD = jbod1
+	return formatConfigs
+}
+
+// Wrapper for calling FormatXL tests - currently validates
+//  - valid format
+//  - unrecognized version number
+//  - invalid JBOD
+//  - invalid Disk uuid
+func TestFormatXL(t *testing.T) {
+	formatInputCases := [][]*formatConfigV1{
+		genFormatXLValid(),
+		genFormatXLInvalidVersion(),
+		genFormatXLInvalidJBOD(),
+		genFormatXLInvalidDisks(),
+		genFormatXLInvalidDisksOrder(),
+	}
+	testCases := []struct {
+		formatConfigs []*formatConfigV1
+		shouldPass    bool
+	}{
+		{
+			formatConfigs: formatInputCases[0],
+			shouldPass:    true,
+		},
+		{
+			formatConfigs: formatInputCases[1],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[2],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[3],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[4],
+			shouldPass:    false,
+		},
+	}
+
+	for i, testCase := range testCases {
+		err := checkFormatXL(testCase.formatConfigs)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass but failed with %s", i+1, err)
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail but passed instead", i+1)
+		}
+	}
+}
+
+// Tests uuid order verification function.
+func TestSavedUUIDOrder(t *testing.T) {
+	uuidTestCases := make([]struct {
+		uuid       string
+		shouldPass bool
+	}, 8)
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+		uuidTestCases[index].uuid = jbod[index]
+		uuidTestCases[index].shouldPass = true
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Re order jbod for failure case.
+	var jbod1 = make([]string, 8)
+	for i, j := range jbod {
+		jbod1[i] = j
+	}
+	jbod1[1], jbod1[2] = jbod[2], jbod[1]
+	formatConfigs[2].XL.JBOD = jbod1
+	uuidTestCases[1].shouldPass = false
+	uuidTestCases[2].shouldPass = false
+
+	for i, testCase := range uuidTestCases {
+		// Is uuid present on all JBOD ?.
+		if testCase.shouldPass != isSavedUUIDInOrder(testCase.uuid, formatConfigs) {
+			t.Errorf("Test %d: Expected to pass but failed", i+1)
+		}
+	}
+}

fs-v1-errors.go

@@ -0,0 +1,22 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "errors"
+
+// errFSDiskFormat - returned when given disk format is other than FS format.
+var errFSDiskFormat = errors.New("Disk is not in FS format.")

fs-v1-metadata.go

@@ -7,7 +7,8 @@ import (
 )
 
 const (
-	fsMetaJSONFile = "fs.json"
+	fsMetaJSONFile   = "fs.json"
+	fsFormatJSONFile = "format.json"
 )
 
 // A fsMetaV1 represents a metadata header mapping keys to sets of values.
@@ -56,40 +57,68 @@ func (m *fsMetaV1) AddObjectPart(partNumber int, partName string, partETag strin
 }
 
 // readFSMetadata - returns the object metadata `fs.json` content.
-func (fs fsObjects) readFSMetadata(bucket, object string) (fsMeta fsMetaV1, err error) {
-	var buffer []byte
-	buffer, err = readAll(fs.storage, bucket, path.Join(object, fsMetaJSONFile))
+func readFSMetadata(disk StorageAPI, bucket, object string) (fsMeta fsMetaV1, err error) {
+	// Read all `fs.json`.
+	buf, err := disk.ReadAll(bucket, path.Join(object, fsMetaJSONFile))
 	if err != nil {
 		return fsMetaV1{}, err
 	}
-	err = json.Unmarshal(buffer, &fsMeta)
-	if err != nil {
+
+	// Decode `fs.json` into fsMeta structure.
+	if err = json.Unmarshal(buf, &fsMeta); err != nil {
 		return fsMetaV1{}, err
 	}
+
+	// Success.
 	return fsMeta, nil
 }
 
 // newFSMetaV1 - initializes new fsMetaV1.
 func newFSMetaV1() (fsMeta fsMetaV1) {
 	fsMeta = fsMetaV1{}
-	fsMeta.Version = "1"
+	fsMeta.Version = "1.0.0"
 	fsMeta.Format = "fs"
 	fsMeta.Minio.Release = minioReleaseTag
 	return fsMeta
 }
 
+// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
+func newFSFormatV1() (format formatConfigV1) {
+	return formatConfigV1{
+		Version: "1",
+		Format:  "fs",
+		FS: &fsFormat{
+			Version: "1",
+		},
+	}
+}
+
+// isFSFormat - returns whether given formatConfigV1 is FS type or not.
+func isFSFormat(format formatConfigV1) bool {
+	return format.Format == "fs"
+}
+
+// writes FS format (format.json) into minioMetaBucket.
+func writeFSFormatData(storage StorageAPI, fsFormat formatConfigV1) error {
+	metadataBytes, err := json.Marshal(fsFormat)
+	if err != nil {
+		return err
+	}
+	// fsFormatJSONFile - format.json file stored in minioMetaBucket(.minio) directory.
+	if err = storage.AppendFile(minioMetaBucket, fsFormatJSONFile, metadataBytes); err != nil {
+		return err
+	}
+	return nil
+}
+
 // writeFSMetadata - writes `fs.json` metadata.
-func (fs fsObjects) writeFSMetadata(bucket, prefix string, fsMeta fsMetaV1) error {
+func (fs fsObjects) writeTempFSMetadata(bucket, path string, fsMeta fsMetaV1) error {
 	metadataBytes, err := json.Marshal(fsMeta)
 	if err != nil {
 		return err
 	}
-	n, err := fs.storage.AppendFile(bucket, path.Join(prefix, fsMetaJSONFile), metadataBytes)
-	if err != nil {
+	if err = fs.storage.AppendFile(bucket, path, metadataBytes); err != nil {
 		return err
 	}
-	if n != int64(len(metadataBytes)) {
-		return errUnexpected
-	}
 	return nil
 }

fs-v1-multipart-common.go

@@ -17,8 +17,10 @@
 package main
 
 import (
+	"encoding/json"
 	"path"
 	"strings"
+	"time"
 )
 
 // Returns if the prefix is a multipart upload.
@@ -68,3 +70,58 @@ func (fs fsObjects) isUploadIDExists(bucket, object, uploadID string) bool {
 	}
 	return true
 }
+
+// writeUploadJSON - create `uploads.json` or update it with new uploadID.
+func (fs fsObjects) writeUploadJSON(bucket, object, uploadID string, initiated time.Time) (err error) {
+	uploadsPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
+	uniqueID := getUUID()
+	tmpUploadsPath := path.Join(tmpMetaPrefix, uniqueID)
+	var uploadsJSON uploadsV1
+	uploadsJSON, err = readUploadsJSON(bucket, object, fs.storage)
+	if err != nil {
+		// For any other errors.
+		if err != errFileNotFound {
+			return err
+		}
+		// Set uploads format to `fs`.
+		uploadsJSON = newUploadsV1("fs")
+	}
+	// Add a new upload id.
+	uploadsJSON.AddUploadID(uploadID, initiated)
+
+	// Update `uploads.json` on all disks.
+	uploadsJSONBytes, wErr := json.Marshal(&uploadsJSON)
+	if wErr != nil {
+		return wErr
+	}
+	// Write `uploads.json` to disk.
+	if wErr = fs.storage.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsJSONBytes); wErr != nil {
+		return wErr
+	}
+	wErr = fs.storage.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath)
+	if wErr != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tmpUploadsPath); dErr != nil {
+			return dErr
+		}
+		return wErr
+	}
+	return nil
+}
+
+// updateUploadsJSON - update `uploads.json` with new uploadsJSON for all disks.
+func (fs fsObjects) updateUploadsJSON(bucket, object string, uploadsJSON uploadsV1) (err error) {
+	uploadsPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
+	uniqueID := getUUID()
+	tmpUploadsPath := path.Join(tmpMetaPrefix, uniqueID)
+	uploadsBytes, wErr := json.Marshal(uploadsJSON)
+	if wErr != nil {
+		return wErr
+	}
+	if wErr = fs.storage.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsBytes); wErr != nil {
+		return wErr
+	}
+	if wErr = fs.storage.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath); wErr != nil {
+		return wErr
+	}
+	return nil
+}

fs-v1-multipart.go

@@ -66,15 +66,17 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 		}
 		maxUploads = maxUploads - len(uploads)
 	}
+	var walkResultCh chan treeWalkResult
+	var endWalkCh chan struct{}
 	if maxUploads > 0 {
-		walker := fs.lookupTreeWalk(listParams{minioMetaBucket, recursive, multipartMarkerPath, multipartPrefixPath})
-		if walker == nil {
-			walker = fs.startTreeWalk(minioMetaBucket, multipartPrefixPath, multipartMarkerPath, recursive, func(bucket, object string) bool {
-				return fs.isMultipartUpload(bucket, object)
-			})
+		walkResultCh, endWalkCh = fs.listPool.Release(listParams{minioMetaBucket, recursive, multipartMarkerPath, multipartPrefixPath})
+		if walkResultCh == nil {
+			endWalkCh = make(chan struct{})
+			listDir := listDirFactory(fs.isMultipartUpload, fs.storage)
+			walkResultCh = startTreeWalk(minioMetaBucket, multipartPrefixPath, multipartMarkerPath, recursive, listDir, endWalkCh)
 		}
 		for maxUploads > 0 {
-			walkResult, ok := <-walker.ch
+			walkResult, ok := <-walkResultCh
 			if !ok {
 				// Closed channel.
 				eof = true
@@ -83,7 +85,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 			// For any walk error return right away.
 			if walkResult.err != nil {
 				// File not found or Disk not found is a valid case.
-				if walkResult.err == errFileNotFound || walkResult.err == errDiskNotFound {
+				if isErrIgnored(walkResult.err, walkResultIgnoredErrs) {
 					eof = true
 					break
 				}
@@ -137,6 +139,13 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 		result.NextKeyMarker = objectName
 		result.NextUploadIDMarker = uploadID
 	}
+
+	if !eof {
+		// Save the go-routine state in the pool so that it can continue from where it left off on
+		// the next request.
+		fs.listPool.Set(listParams{bucket, recursive, result.NextKeyMarker, prefix}, walkResultCh, endWalkCh)
+	}
+
 	result.IsTruncated = !eof
 	if !result.IsTruncated {
 		result.NextKeyMarker = ""
@@ -206,11 +215,6 @@ func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 // all the disks. `uploads.json` carries metadata regarding on going
 // multipart operation on the object.
 func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[string]string) (uploadID string, err error) {
-	// No metadata is set, allocate a new one.
-	if meta == nil {
-		meta = make(map[string]string)
-	}
-
 	// Initialize `fs.json` values.
 	fsMeta := newFSMetaV1()
 
@@ -221,18 +225,18 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st
 	uploadID = getUUID()
 	initiated := time.Now().UTC()
 	// Create 'uploads.json'
-	if err = writeUploadJSON(bucket, object, uploadID, initiated, fs.storage); err != nil {
+	if err = fs.writeUploadJSON(bucket, object, uploadID, initiated); err != nil {
 		return "", err
 	}
 	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
-	tempUploadIDPath := path.Join(tmpMetaPrefix, uploadID)
-	if err = fs.writeFSMetadata(minioMetaBucket, tempUploadIDPath, fsMeta); err != nil {
-		return "", toObjectErr(err, minioMetaBucket, tempUploadIDPath)
+	tempFSMetadataPath := path.Join(tmpMetaPrefix, getUUID()+"-"+fsMetaJSONFile)
+	if err = fs.writeTempFSMetadata(minioMetaBucket, tempFSMetadataPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, tempFSMetadataPath)
 	}
-	err = fs.storage.RenameFile(minioMetaBucket, path.Join(tempUploadIDPath, fsMetaJSONFile), minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
+	err = fs.storage.RenameFile(minioMetaBucket, tempFSMetadataPath, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
 	if err != nil {
-		if dErr := fs.storage.DeleteFile(minioMetaBucket, path.Join(tempUploadIDPath, fsMetaJSONFile)); dErr != nil {
-			return "", toObjectErr(dErr, minioMetaBucket, tempUploadIDPath)
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempFSMetadataPath); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tempFSMetadataPath)
 		}
 		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
 	}
@@ -294,14 +298,32 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(partID)))
 
 	partSuffix := fmt.Sprintf("object%d", partID)
-	tmpPartPath := path.Join(tmpMetaPrefix, uploadID, partSuffix)
+	tmpPartPath := path.Join(tmpMetaPrefix, uploadID+"-"+partSuffix)
 
 	// Initialize md5 writer.
 	md5Writer := md5.New()
 
-	var buf = make([]byte, blockSizeV1)
+	// Limit the reader to its provided size if specified.
+	var limitDataReader io.Reader
+	if size > 0 {
+		// This is done so that we can avoid erroneous clients sending more data than the set content size.
+		limitDataReader = io.LimitReader(data, size)
+	} else {
+		// else we read till EOF.
+		limitDataReader = data
+	}
+
+	// Allocate buffer for staging buffer.
+	bufSize := int64(readSizeV1)
+	if size > 0 && bufSize > size {
+		bufSize = size
+	}
+	var buf = make([]byte, int(bufSize))
+
+	// Read up to required size
+	totalLeft := size
 	for {
-		n, err := io.ReadFull(data, buf)
+		n, err := io.ReadFull(limitDataReader, buf)
 		if err == io.EOF {
 			break
 		}
@@ -310,18 +332,31 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		}
 		// Update md5 writer.
 		md5Writer.Write(buf[:n])
-		m, err := fs.storage.AppendFile(minioMetaBucket, tmpPartPath, buf[:n])
-		if err != nil {
+		if err = fs.storage.AppendFile(minioMetaBucket, tmpPartPath, buf[:n]); err != nil {
 			return "", toObjectErr(err, bucket, object)
 		}
-		if m != int64(len(buf[:n])) {
-			return "", toObjectErr(errUnexpected, bucket, object)
+
+		if totalLeft -= int64(n); size >= 0 && totalLeft <= 0 {
+			break
+		}
+	}
+
+	// Validate if payload is valid.
+	if isSignVerify(data) {
+		if err := data.(*signVerifyReader).Verify(); err != nil {
+			// Incoming payload wrong, delete the temporary object.
+			fs.storage.DeleteFile(minioMetaBucket, tmpPartPath)
+			// Error return.
+			return "", toObjectErr(err, bucket, object)
 		}
 	}
 
 	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
 	if md5Hex != "" {
 		if newMD5Hex != md5Hex {
+			// MD5 mismatch, delete the temporary object.
+			fs.storage.DeleteFile(minioMetaBucket, tmpPartPath)
+			// Returns md5 mismatch.
 			return "", BadDigest{md5Hex, newMD5Hex}
 		}
 	}
@@ -335,7 +370,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		return "", InvalidUploadID{UploadID: uploadID}
 	}
 
-	fsMeta, err := fs.readFSMetadata(minioMetaBucket, uploadIDPath)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
 	if err != nil {
 		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
 	}
@@ -350,14 +385,14 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		return "", toObjectErr(err, minioMetaBucket, partPath)
 	}
 	uploadIDPath = path.Join(mpartMetaPrefix, bucket, object, uploadID)
-	tempUploadIDPath := path.Join(tmpMetaPrefix, uploadID)
-	if err = fs.writeFSMetadata(minioMetaBucket, tempUploadIDPath, fsMeta); err != nil {
-		return "", toObjectErr(err, minioMetaBucket, tempUploadIDPath)
+	tempFSMetadataPath := path.Join(tmpMetaPrefix, getUUID()+"-"+fsMetaJSONFile)
+	if err = fs.writeTempFSMetadata(minioMetaBucket, tempFSMetadataPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, tempFSMetadataPath)
 	}
-	err = fs.storage.RenameFile(minioMetaBucket, path.Join(tempUploadIDPath, fsMetaJSONFile), minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
+	err = fs.storage.RenameFile(minioMetaBucket, tempFSMetadataPath, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
 	if err != nil {
-		if dErr := fs.storage.DeleteFile(minioMetaBucket, path.Join(tempUploadIDPath, fsMetaJSONFile)); dErr != nil {
-			return "", toObjectErr(dErr, minioMetaBucket, tempUploadIDPath)
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempFSMetadataPath); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tempFSMetadataPath)
 		}
 		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
 	}
@@ -371,7 +406,7 @@ func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberM
 	result := ListPartsInfo{}
 
 	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
-	fsMeta, err := fs.readFSMetadata(minioMetaBucket, uploadIDPath)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
 	if err != nil {
 		return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, uploadIDPath)
 	}
@@ -479,7 +514,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	}
 
 	// Read saved fs metadata for ongoing multipart.
-	fsMeta, err := fs.readFSMetadata(minioMetaBucket, uploadIDPath)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
 	if err != nil {
 		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
 	}
@@ -490,8 +525,10 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		return "", err
 	}
 
-	tempObj := path.Join(tmpMetaPrefix, uploadID, "object1")
-	var buffer = make([]byte, blockSizeV1)
+	tempObj := path.Join(tmpMetaPrefix, uploadID+"-"+"part.1")
+
+	// Allocate staging buffer.
+	var buf = make([]byte, readSizeV1)
 
 	// Loop through all parts, validate them and then commit to disk.
 	for i, part := range parts {
@@ -504,7 +541,11 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		}
 		// All parts except the last part has to be atleast 5MB.
 		if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
-			return "", PartTooSmall{}
+			return "", PartTooSmall{
+				PartNumber: part.PartNumber,
+				PartSize:   fsMeta.Parts[partIdx].Size,
+				PartETag:   part.ETag,
+			}
 		}
 		// Construct part suffix.
 		partSuffix := fmt.Sprintf("object%d", part.PartNumber)
@@ -512,18 +553,26 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		offset := int64(0)
 		totalLeft := fsMeta.Parts[partIdx].Size
 		for totalLeft > 0 {
+			curLeft := int64(readSizeV1)
+			if totalLeft < readSizeV1 {
+				curLeft = totalLeft
+			}
 			var n int64
-			n, err = fs.storage.ReadFile(minioMetaBucket, multipartPartFile, offset, buffer)
+			n, err = fs.storage.ReadFile(minioMetaBucket, multipartPartFile, offset, buf[:curLeft])
+			if n > 0 {
+				if err = fs.storage.AppendFile(minioMetaBucket, tempObj, buf[:n]); err != nil {
+					return "", toObjectErr(err, minioMetaBucket, tempObj)
+				}
+			}
 			if err != nil {
+				if err == io.EOF || err == io.ErrUnexpectedEOF {
+					break
+				}
 				if err == errFileNotFound {
 					return "", InvalidPart{}
 				}
 				return "", toObjectErr(err, minioMetaBucket, multipartPartFile)
 			}
-			n, err = fs.storage.AppendFile(minioMetaBucket, tempObj, buffer[:n])
-			if err != nil {
-				return "", toObjectErr(err, minioMetaBucket, tempObj)
-			}
 			offset += n
 			totalLeft -= n
 		}
@@ -561,7 +610,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		uploadsJSON.Uploads = append(uploadsJSON.Uploads[:uploadIDIdx], uploadsJSON.Uploads[uploadIDIdx+1:]...)
 	}
 	if len(uploadsJSON.Uploads) > 0 {
-		if err = updateUploadsJSON(bucket, object, uploadsJSON, fs.storage); err != nil {
+		if err = fs.updateUploadsJSON(bucket, object, uploadsJSON); err != nil {
 			return "", toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object))
 		}
 		// Return success.
@@ -597,7 +646,7 @@ func (fs fsObjects) abortMultipartUpload(bucket, object, uploadID string) error
 		// There are pending uploads for the same object, preserve
 		// them update 'uploads.json' in-place.
 		if len(uploadsJSON.Uploads) > 0 {
-			err = updateUploadsJSON(bucket, object, uploadsJSON, fs.storage)
+			err = fs.updateUploadsJSON(bucket, object, uploadsJSON)
 			if err != nil {
 				return toObjectErr(err, bucket, object)
 			}

fs-v1.go

@@ -19,13 +19,13 @@ package main
 import (
 	"crypto/md5"
 	"encoding/hex"
+	"encoding/json"
 	"io"
 	"os"
 	"path"
 	"path/filepath"
 	"sort"
 	"strings"
-	"sync"
 
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/mimedb"
@@ -33,28 +33,101 @@ import (
 
 // fsObjects - Implements fs object layer.
 type fsObjects struct {
-	storage            StorageAPI
-	physicalDisk       string
-	listObjectMap      map[listParams][]*treeWalkerFS
-	listObjectMapMutex *sync.Mutex
+	storage      StorageAPI
+	physicalDisk string
+
+	// List pool management.
+	listPool *treeWalkPool
+}
+
+// creates format.json, the FS format info in minioMetaBucket.
+func initFormatFS(storageDisk StorageAPI) error {
+	return writeFSFormatData(storageDisk, newFSFormatV1())
+}
+
+// loads format.json from minioMetaBucket if it exists.
+func loadFormatFS(storageDisk StorageAPI) (format formatConfigV1, err error) {
+	// Reads entire `format.json`.
+	buf, err := storageDisk.ReadAll(minioMetaBucket, fsFormatJSONFile)
+	if err != nil {
+		return formatConfigV1{}, err
+	}
+
+	// Unmarshal format config.
+	if err = json.Unmarshal(buf, &format); err != nil {
+		return formatConfigV1{}, err
+	}
+
+	// Return structured `format.json`.
+	return format, nil
+}
+
+// Should be called when process shuts down.
+func shutdownFS(storage StorageAPI) {
+	_, err := storage.ListDir(minioMetaBucket, mpartMetaPrefix)
+	if err != errFileNotFound {
+		// Multipart directory is not empty hence do not remove .minio volume.
+		os.Exit(0)
+	}
+	prefix := ""
+	if err := cleanupDir(storage, minioMetaBucket, prefix); err != nil {
+		os.Exit(0)
+		return
+	}
+	storage.DeleteVol(minioMetaBucket)
+	os.Exit(0)
 }
 
 // newFSObjects - initialize new fs object layer.
 func newFSObjects(disk string) (ObjectLayer, error) {
 	storage, err := newStorageAPI(disk)
-	if err != nil {
+	if err != nil && err != errDiskNotFound {
 		return nil, err
 	}
 
+	// Attempt to create `.minio`.
+	err = storage.MakeVol(minioMetaBucket)
+	if err != nil {
+		switch err {
+		// Ignore the errors.
+		case errVolumeExists, errDiskNotFound, errFaultyDisk:
+		default:
+			return nil, toObjectErr(err, minioMetaBucket)
+		}
+	}
+
 	// Runs house keeping code, like creating minioMetaBucket, cleaning up tmp files etc.
-	fsHouseKeeping(storage)
+	if err = fsHouseKeeping(storage); err != nil {
+		return nil, err
+	}
+
+	// loading format.json from minioMetaBucket.
+	// Note: The format.json content is ignored, reserved for future use.
+	format, err := loadFormatFS(storage)
+	if err != nil {
+		if err == errFileNotFound {
+			// format.json doesn't exist, create it inside minioMetaBucket.
+			err = initFormatFS(storage)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	} else if !isFSFormat(format) {
+		return nil, errFSDiskFormat
+	}
+
+	// Register the callback that should be called when the process shuts down.
+	registerShutdown(func() {
+		shutdownFS(storage)
+	})
 
 	// Return successfully initialized object layer.
 	return fsObjects{
-		storage:            storage,
-		physicalDisk:       disk,
-		listObjectMap:      make(map[listParams][]*treeWalkerFS),
-		listObjectMapMutex: &sync.Mutex{},
+		storage:      storage,
+		physicalDisk: disk,
+		listPool:     newTreeWalkPool(globalLookupTimeout),
 	}, nil
 }
 
@@ -126,16 +199,21 @@ func (fs fsObjects) DeleteBucket(bucket string) error {
 	if !IsValidBucketName(bucket) {
 		return BucketNameInvalid{Bucket: bucket}
 	}
+	// Attempt to delete regular bucket.
 	if err := fs.storage.DeleteVol(bucket); err != nil {
 		return toObjectErr(err, bucket)
 	}
+	// Cleanup all the previously incomplete multiparts.
+	if err := cleanupDir(fs.storage, path.Join(minioMetaBucket, mpartMetaPrefix), bucket); err != nil && err != errVolumeNotFound {
+		return toObjectErr(err, bucket)
+	}
 	return nil
 }
 
 /// Object Operations
 
 // GetObject - get an object.
-func (fs fsObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
+func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer) (err error) {
 	// Verify if bucket is valid.
 	if !IsValidBucketName(bucket) {
 		return BucketNameInvalid{Bucket: bucket}
@@ -144,28 +222,77 @@ func (fs fsObjects) GetObject(bucket, object string, startOffset int64, length i
 	if !IsValidObjectName(object) {
 		return ObjectNameInvalid{Bucket: bucket, Object: object}
 	}
-	var totalLeft = length
-	for totalLeft > 0 {
-		// Figure out the right blockSize as it was encoded before.
-		var curBlockSize int64
-		if blockSizeV1 < totalLeft {
-			curBlockSize = blockSizeV1
-		} else {
-			curBlockSize = totalLeft
-		}
-		buf := make([]byte, curBlockSize)
-		n, err := fs.storage.ReadFile(bucket, object, startOffset, buf)
-		if err != nil {
-			return toObjectErr(err, bucket, object)
-		}
-		_, err = writer.Write(buf[:n])
-		if err != nil {
-			return toObjectErr(err, bucket, object)
-		}
-		totalLeft -= n
-		startOffset += n
+	// Offset and length cannot be negative.
+	if offset < 0 || length < 0 {
+		return toObjectErr(errUnexpected, bucket, object)
 	}
-	return nil
+	// Writer cannot be nil.
+	if writer == nil {
+		return toObjectErr(errUnexpected, bucket, object)
+	}
+
+	// Stat the file to get file size.
+	fi, err := fs.storage.StatFile(bucket, object)
+	if err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	// Reply back invalid range if the input offset and length fall out of range.
+	if offset > fi.Size || length > fi.Size {
+		return InvalidRange{offset, length, fi.Size}
+	}
+	// Reply if we have inputs with offset and length falling out of file size range.
+	if offset+length > fi.Size {
+		return InvalidRange{offset, length, fi.Size}
+	}
+
+	var totalLeft = length
+	bufSize := int64(readSizeV1)
+	if length > 0 && bufSize > length {
+		bufSize = length
+	}
+	// Allocate a staging buffer.
+	buf := make([]byte, int(bufSize))
+	for {
+		// Figure out the right size for the buffer.
+		curLeft := bufSize
+		if totalLeft < bufSize {
+			curLeft = totalLeft
+		}
+		// Reads the file at offset.
+		nr, er := fs.storage.ReadFile(bucket, object, offset, buf[:curLeft])
+		if nr > 0 {
+			// Write to response writer.
+			nw, ew := writer.Write(buf[0:nr])
+			if nw > 0 {
+				// Decrement whats left to write.
+				totalLeft -= int64(nw)
+
+				// Progress the offset
+				offset += int64(nw)
+			}
+			if ew != nil {
+				err = ew
+				break
+			}
+			if nr != int64(nw) {
+				err = io.ErrShortWrite
+				break
+			}
+		}
+		if er == io.EOF || er == io.ErrUnexpectedEOF {
+			break
+		}
+		if er != nil {
+			err = er
+			break
+		}
+		if totalLeft == 0 {
+			break
+		}
+	}
+	// Returns any error.
+	return toObjectErr(err, bucket, object)
 }
 
 // GetObjectInfo - get object info.
@@ -182,13 +309,15 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
 	if err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
-	contentType := "application/octet-stream"
+
+	// Guess content-type from the extension if possible.
+	contentType := ""
 	if objectExt := filepath.Ext(object); objectExt != "" {
-		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
-		if ok {
+		if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
 			contentType = content.ContentType
 		}
 	}
+
 	return ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
@@ -215,37 +344,54 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 
 	uniqueID := getUUID()
 
-	// Temporary object.
+	// Uploaded object will first be written to the temporary location which will eventually
+	// be renamed to the actual location. It is first written to the temporary location
+	// so that cleaning it up will be easy if the server goes down.
 	tempObj := path.Join(tmpMetaPrefix, uniqueID)
 
 	// Initialize md5 writer.
 	md5Writer := md5.New()
 
+	// Limit the reader to its provided size if specified.
+	var limitDataReader io.Reader
+	if size > 0 {
+		// This is done so that we can avoid erroneous clients sending more data than the set content size.
+		limitDataReader = io.LimitReader(data, size)
+	} else {
+		// else we read till EOF.
+		limitDataReader = data
+	}
+
 	if size == 0 {
 		// For size 0 we write a 0byte file.
-		_, err := fs.storage.AppendFile(minioMetaBucket, tempObj, []byte(""))
+		err := fs.storage.AppendFile(minioMetaBucket, tempObj, []byte(""))
 		if err != nil {
 			return "", toObjectErr(err, bucket, object)
 		}
 	} else {
-		// Allocate buffer.
-		buf := make([]byte, blockSizeV1)
+		// Allocate a buffer to Read() the object upload stream.
+		bufSize := int64(readSizeV1)
+		if size > 0 && bufSize > size {
+			bufSize = size
+		}
+		buf := make([]byte, int(bufSize))
+
+		// Read the buffer till io.EOF and append the read data to the temporary file.
 		for {
-			n, rErr := data.Read(buf)
-			if rErr == io.EOF {
-				break
-			}
-			if rErr != nil {
+			n, rErr := limitDataReader.Read(buf)
+			if rErr != nil && rErr != io.EOF {
 				return "", toObjectErr(rErr, bucket, object)
 			}
-			// Update md5 writer.
-			md5Writer.Write(buf[:n])
-			m, wErr := fs.storage.AppendFile(minioMetaBucket, tempObj, buf[:n])
-			if wErr != nil {
-				return "", toObjectErr(wErr, bucket, object)
+			if n > 0 {
+				// Update md5 writer.
+				md5Writer.Write(buf[0:n])
+				wErr := fs.storage.AppendFile(minioMetaBucket, tempObj, buf[0:n])
+				if wErr != nil {
+					return "", toObjectErr(wErr, bucket, object)
+				}
 			}
-			if m != int64(len(buf[:n])) {
-				return "", toObjectErr(errUnexpected, bucket, object)
+			if rErr == io.EOF {
+				break
 			}
 		}
 	}
@@ -256,12 +402,27 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 	if len(metadata) != 0 {
 		md5Hex = metadata["md5Sum"]
 	}
+
+	// Validate if payload is valid.
+	if isSignVerify(data) {
+		if vErr := data.(*signVerifyReader).Verify(); vErr != nil {
+			// Incoming payload wrong, delete the temporary object.
+			fs.storage.DeleteFile(minioMetaBucket, tempObj)
+			// Error return.
+			return "", toObjectErr(vErr, bucket, object)
+		}
+	}
+
 	if md5Hex != "" {
 		if newMD5Hex != md5Hex {
+			// MD5 mismatch, delete the temporary object.
+			fs.storage.DeleteFile(minioMetaBucket, tempObj)
+			// Returns md5 mismatch.
 			return "", BadDigest{md5Hex, newMD5Hex}
 		}
 	}
 
+	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
 	err := fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
 	if err != nil {
 		return "", toObjectErr(err, bucket, object)
@@ -368,17 +529,19 @@ func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey
 		recursive = false
 	}
 
-	walker := fs.lookupTreeWalk(listParams{bucket, recursive, marker, prefix})
-	if walker == nil {
-		walker = fs.startTreeWalk(bucket, prefix, marker, recursive, func(bucket, object string) bool {
+	walkResultCh, endWalkCh := fs.listPool.Release(listParams{bucket, recursive, marker, prefix})
+	if walkResultCh == nil {
+		endWalkCh = make(chan struct{})
+		listDir := listDirFactory(func(bucket, object string) bool {
 			return !strings.HasSuffix(object, slashSeparator)
-		})
+		}, fs.storage)
+		walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, endWalkCh)
 	}
 	var fileInfos []FileInfo
 	var eof bool
 	var nextMarker string
 	for i := 0; i < maxKeys; {
-		walkResult, ok := <-walker.ch
+		walkResult, ok := <-walkResultCh
 		if !ok {
 			// Closed channel.
 			eof = true
@@ -406,7 +569,7 @@ func (fs fsObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey
 	}
 	params := listParams{bucket, recursive, nextMarker, prefix}
 	if !eof {
-		fs.saveTreeWalk(params, walker)
+		fs.listPool.Set(params, walkResultCh, endWalkCh)
 	}
 
 	result := ListObjectsInfo{IsTruncated: !eof}

fs-v1_test.go

@@ -0,0 +1,61 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestNewFS - tests initialization of all input disks
+// and constructs a valid `FS` object layer.
+func TestNewFS(t *testing.T) {
+	// Do not attempt to create this path, the test validates
+	// so that newFSObjects initializes non existing paths
+	// and successfully returns initialized object layer.
+	disk := filepath.Join(os.TempDir(), "minio-"+nextSuffix())
+	defer removeAll(disk)
+
+	// Setup to test errFSDiskFormat.
+	disks := []string{}
+	for i := 0; i < 6; i++ {
+		xlDisk := filepath.Join(os.TempDir(), "minio-"+nextSuffix())
+		defer removeAll(xlDisk)
+		disks = append(disks, xlDisk)
+	}
+
+	// Initializes all disks with XL
+	_, err := newXLObjects(disks, nil)
+	if err != nil {
+		t.Fatalf("Unable to initialize XL object, %s", err)
+	}
+
+	testCases := []struct {
+		disk        string
+		expectedErr error
+	}{
+		{disk, nil},
+		{disks[0], errFSDiskFormat},
+	}
+
+	for _, testCase := range testCases {
+		if _, err := newFSObjects(testCase.disk); err != testCase.expectedErr {
+			t.Fatalf("expected: %s, got: %s", testCase.expectedErr, err)
+		}
+	}
+}

generic-handlers.go

@@ -141,7 +141,6 @@ func parseAmzDate(amzDateStr string) (amzDate time.Time, apiErr APIErrorCode) {
 // Supported Amz date headers.
 var amzDateHeaders = []string{
 	"x-amz-date",
-	"x-minio-date",
 	"date",
 }
 

globals.go

@@ -16,7 +16,10 @@
 
 package main
 
-import "github.com/fatih/color"
+import (
+	"github.com/fatih/color"
+	"github.com/minio/minio/pkg/objcache"
+)
 
 // Global constants for Minio.
 const (
@@ -27,7 +30,7 @@ const (
 const (
 	globalMinioConfigVersion = "4"
 	globalMinioConfigDir     = ".minio"
-	globalMinioCertsDir      = ".minio/certs"
+	globalMinioCertsDir      = "certs"
 	globalMinioCertFile      = "public.crt"
 	globalMinioKeyFile       = "private.key"
 	globalMinioConfigFile    = "config.json"
@@ -39,11 +42,21 @@ var (
 	globalQuiet = false // Quiet flag set via command line
 	globalTrace = false // Trace flag set via environment setting.
 	// Add new global flags here.
+
+	// Maximum connections handled per
+	// server, defaults to 0 (unlimited).
+	globalMaxConn = 0
+	// Maximum cache size.
+	globalMaxCacheSize = uint64(maxCacheSize)
+	// Cache expiry.
+	globalCacheExpiry = objcache.DefaultExpiry
+	// Add new variable global values here.
 )
 
 // global colors.
 var (
-	colorMagenta = color.New(color.FgMagenta, color.Bold).SprintfFunc()
-	colorWhite   = color.New(color.FgWhite, color.Bold).SprintfFunc()
-	colorGreen   = color.New(color.FgGreen, color.Bold).SprintfFunc()
+	colorWhite     = color.New(color.FgWhite).SprintfFunc()
+	colorWhiteBold = color.New(color.FgWhite, color.Bold).SprintfFunc()
+	colorBlue      = color.New(color.FgBlue).SprintfFunc()
+	colorBold      = color.New(color.Bold).SprintFunc()
 )

host-to-ip.go

@@ -0,0 +1,47 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"net"
+	"sort"
+)
+
+// byLastOctet implements sort.Interface used in sorting a list
+// of ip address by their last octet value.
+type byLastOctet []net.IP
+
+func (n byLastOctet) Len() int      { return len(n) }
+func (n byLastOctet) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
+func (n byLastOctet) Less(i, j int) bool {
+	return []byte(n[i].To4())[3] < []byte(n[j].To4())[3]
+}
+
+// getIPsFromHosts - returns a reverse sorted list of ips based on the last octet value.
+func getIPsFromHosts(hosts []string) (ips []net.IP) {
+	for _, host := range hosts {
+		ips = append(ips, net.ParseIP(host))
+	}
+	// Reverse sort ips by their last octet.
+	sort.Sort(sort.Reverse(byLastOctet(ips)))
+	return ips
+}
+
+// getHostToIP - parses a host string into net.IP value.
+func getHostToIP(host string) net.IP {
+	return net.ParseIP(host)
+}

host-to-ip_test.go

@@ -0,0 +1,61 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "testing"
+
+// Tests sorted list generated from hosts to ip.
+func TestHostToIP(t *testing.T) {
+	// Collection of test cases to validate last octet sorting.
+	testCases := []struct {
+		hosts       []string
+		sortedHosts []string
+	}{
+		{
+			// List of ip addresses that need to be sorted.
+			[]string{
+				"129.95.30.40",
+				"5.24.69.2",
+				"19.20.203.5",
+				"1.2.3.4",
+				"127.0.0.1",
+				"19.20.21.22",
+				"5.220.100.50",
+			},
+			// Numerical sorting result based on the last octet.
+			[]string{
+				"5.220.100.50",
+				"129.95.30.40",
+				"19.20.21.22",
+				"19.20.203.5",
+				"1.2.3.4",
+				"5.24.69.2",
+				"127.0.0.1",
+			},
+		},
+	}
+
+	// Tests the correct sorting behavior of getIPsFromHosts.
+	for j, testCase := range testCases {
+		ips := getIPsFromHosts(testCase.hosts)
+		for i, ip := range ips {
+			if ip.String() != testCase.sortedHosts[i] {
+				t.Fatalf("Test %d expected to pass but failed. Wanted ip %s, but got %s", j+1, testCase.sortedHosts[i], ip.String())
+			}
+		}
+	}
+}

httprange.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,112 +17,118 @@
 package main
 
 import (
-	"errors"
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 )
 
 const (
-	b = "bytes="
+	byteRangePrefix = "bytes="
 )
 
-// InvalidRange - invalid range
-type InvalidRange struct{}
-
-func (e InvalidRange) Error() string {
-	return "The requested range is not satisfiable"
-}
+// Valid byte position regexp
+var validBytePos = regexp.MustCompile(`^[0-9]+$`)
 
 // HttpRange specifies the byte range to be sent to the client.
 type httpRange struct {
-	start, length, size int64
+	offsetBegin  int64
+	offsetEnd    int64
+	resourceSize int64
 }
 
 // String populate range stringer interface
-func (r *httpRange) String() string {
-	return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, r.size)
+func (hrange httpRange) String() string {
+	return fmt.Sprintf("bytes %d-%d/%d", hrange.offsetBegin, hrange.offsetEnd, hrange.resourceSize)
 }
 
-// Grab new range from request header
-func getRequestedRange(hrange string, size int64) (*httpRange, error) {
-	r := &httpRange{
-		start:  0,
-		length: 0,
-		size:   0,
-	}
-	r.size = size
-	if hrange != "" {
-		err := r.parseRange(hrange)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return r, nil
+// getlength - get length from the range.
+func (hrange httpRange) getLength() int64 {
+	return 1 + hrange.offsetEnd - hrange.offsetBegin
 }
 
-func (r *httpRange) parse(ra string) error {
-	i := strings.Index(ra, "-")
-	if i < 0 {
-		return InvalidRange{}
+func parseRequestRange(rangeString string, resourceSize int64) (hrange *httpRange, err error) {
+	// Return error if given range string doesn't start with byte range prefix.
+	if !strings.HasPrefix(rangeString, byteRangePrefix) {
+		return nil, fmt.Errorf("'%s' does not start with '%s'", rangeString, byteRangePrefix)
 	}
-	start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:])
-	if start == "" {
-		// If no start is specified, end specifies the
-		// range start relative to the end of the file.
-		i, err := strconv.ParseInt(end, 10, 64)
-		if err != nil {
-			return InvalidRange{}
+
+	// Trim byte range prefix.
+	byteRangeString := strings.TrimPrefix(rangeString, byteRangePrefix)
+
+	// Check if range string contains delimiter '-', else return error. eg. "bytes=8"
+	sepIndex := strings.Index(byteRangeString, "-")
+	if sepIndex == -1 {
+		return nil, fmt.Errorf("'%s' does not have a valid range value", rangeString)
+	}
+
+	offsetBeginString := byteRangeString[:sepIndex]
+	offsetBegin := int64(-1)
+	// Convert offsetBeginString only if its not empty.
+	if len(offsetBeginString) > 0 {
+		if !validBytePos.MatchString(offsetBeginString) {
+			return nil, fmt.Errorf("'%s' does not have a valid first byte position value", rangeString)
 		}
-		if i > r.size {
-			i = r.size
+
+		if offsetBegin, err = strconv.ParseInt(offsetBeginString, 10, 64); err != nil {
+			return nil, fmt.Errorf("'%s' does not have a valid first byte position value", rangeString)
 		}
-		r.start = r.size - i
-		r.length = r.size - r.start
-	} else {
-		i, err := strconv.ParseInt(start, 10, 64)
-		if err != nil || i > r.size || i < 0 {
-			return InvalidRange{}
+	}
+
+	offsetEndString := byteRangeString[sepIndex+1:]
+	offsetEnd := int64(-1)
+	// Convert offsetEndString only if its not empty.
+	if len(offsetEndString) > 0 {
+		if !validBytePos.MatchString(offsetEndString) {
+			return nil, fmt.Errorf("'%s' does not have a valid last byte position value", rangeString)
 		}
-		r.start = i
-		if end == "" {
-			// If no end is specified, range extends to end of the file.
-			r.length = r.size - r.start
+
+		if offsetEnd, err = strconv.ParseInt(offsetEndString, 10, 64); err != nil {
+			return nil, fmt.Errorf("'%s' does not have a valid last byte position value", rangeString)
+		}
+	}
+
+	// rangeString contains first and last byte positions. eg. "bytes=2-5"
+	if offsetBegin > -1 && offsetEnd > -1 {
+		if offsetBegin > offsetEnd {
+			// Last byte position is not greater than first byte position. eg. "bytes=5-2"
+			return nil, fmt.Errorf("'%s' does not have valid range value", rangeString)
+		}
+
+		// First and last byte positions should not be >= resourceSize.
+		if offsetBegin >= resourceSize {
+			return nil, errInvalidRange
+		}
+
+		if offsetEnd >= resourceSize {
+			offsetEnd = resourceSize - 1
+		}
+	} else if offsetBegin > -1 {
+		// rangeString contains only first byte position. eg. "bytes=8-"
+		if offsetBegin >= resourceSize {
+			// First byte position should not be >= resourceSize.
+			return nil, errInvalidRange
+		}
+
+		offsetEnd = resourceSize - 1
+	} else if offsetEnd > -1 {
+		// rangeString contains only last byte position. eg. "bytes=-3"
+		if offsetEnd == 0 {
+			// Last byte position should not be zero eg. "bytes=-0"
+			return nil, errInvalidRange
+		}
+
+		if offsetEnd >= resourceSize {
+			offsetBegin = 0
 		} else {
-			i, err := strconv.ParseInt(end, 10, 64)
-			if err != nil || r.start > i {
-				return InvalidRange{}
-			}
-			if i >= r.size {
-				i = r.size - 1
-			}
-			r.length = i - r.start + 1
+			offsetBegin = resourceSize - offsetEnd
 		}
+
+		offsetEnd = resourceSize - 1
+	} else {
+		// rangeString contains first and last byte positions missing. eg. "bytes=-"
+		return nil, fmt.Errorf("'%s' does not have valid range value", rangeString)
 	}
-	return nil
-}
-
-// parseRange parses a Range header string as per RFC 2616.
-func (r *httpRange) parseRange(s string) error {
-	if s == "" {
-		return errors.New("header not present")
-	}
-	if !strings.HasPrefix(s, b) {
-		return InvalidRange{}
-	}
-
-	ras := strings.Split(s[len(b):], ",")
-	if len(ras) == 0 {
-		return errors.New("invalid request")
-	}
-	// Just pick the first one and ignore the rest, we only support one range per object
-	if len(ras) > 1 {
-		return errors.New("multiple ranges specified")
-	}
-
-	ra := strings.TrimSpace(ras[0])
-	if ra == "" {
-		return InvalidRange{}
-	}
-	return r.parse(ra)
+
+	return &httpRange{offsetBegin, offsetEnd, resourceSize}, nil
 }

httprange_test.go

@@ -0,0 +1,90 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "testing"
+
+// Test parseRequestRange()
+func TestParseRequestRange(t *testing.T) {
+	// Test success cases.
+	successCases := []struct {
+		rangeString string
+		offsetBegin int64
+		offsetEnd   int64
+		length      int64
+	}{
+		{"bytes=2-5", 2, 5, 4},
+		{"bytes=2-20", 2, 9, 8},
+		{"bytes=2-2", 2, 2, 1},
+		{"bytes=0000-0006", 0, 6, 7},
+		{"bytes=2-", 2, 9, 8},
+		{"bytes=-4", 6, 9, 4},
+		{"bytes=-20", 0, 9, 10},
+	}
+
+	for _, successCase := range successCases {
+		hrange, err := parseRequestRange(successCase.rangeString, 10)
+		if err != nil {
+			t.Fatalf("expected: <nil>, got: %s", err)
+		}
+
+		if hrange.offsetBegin != successCase.offsetBegin {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, hrange.offsetBegin)
+		}
+
+		if hrange.offsetEnd != successCase.offsetEnd {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, hrange.offsetEnd)
+		}
+		if hrange.getLength() != successCase.length {
+			t.Fatalf("expected: %d, got: %d", successCase.length, hrange.getLength())
+		}
+	}
+
+	// Test invalid range strings.
+	invalidRangeStrings := []string{
+		"bytes=8",
+		"bytes=5-2",
+		"bytes=+2-5",
+		"bytes=2-+5",
+		"bytes=2--5",
+		"bytes=-",
+		"",
+		"2-5",
+		"bytes = 2-5",
+		"bytes=2 - 5",
+		"bytes=0-0,-1",
+		"bytes=2-5 ",
+	}
+	for _, rangeString := range invalidRangeStrings {
+		if _, err := parseRequestRange(rangeString, 10); err == nil {
+			t.Fatalf("expected: an error, got: <nil>")
+		}
+	}
+
+	// Test error range strings.
+	errorRangeString := []string{
+		"bytes=10-10",
+		"bytes=20-30",
+		"bytes=20-",
+		"bytes=-0",
+	}
+	for _, rangeString := range errorRangeString {
+		if _, err := parseRequestRange(rangeString, 10); err != errInvalidRange {
+			t.Fatalf("expected: %s, got: %s", errInvalidRange, err)
+		}
+	}
+}

logger-file-hook.go

@@ -39,7 +39,8 @@ func enableFileLogger() {
 		return
 	}
 
-	file, err := os.OpenFile(flogger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	// Creates the named file with mode 0666, honors system umask.
+	file, err := os.OpenFile(flogger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
 	fatalIf(err, "Unable to open log file.")
 
 	// Add a local file hook.

main.go

@@ -19,7 +19,6 @@ package main
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"sort"
 
 	"github.com/minio/cli"
@@ -64,12 +63,6 @@ func init() {
 
 	// Set global trace flag.
 	globalTrace = os.Getenv("MINIO_TRACE") == "1"
-
-	// It is an unsafe practice to run network services as
-	// root. Containers are an exception.
-	if !isContainerized() && os.Geteuid() == 0 {
-		console.Fatalln("Please run ‘minio’ as a non-root user.")
-	}
 }
 
 func migrate() {
@@ -147,11 +140,6 @@ func checkMainSyntax(c *cli.Context) {
 	}
 }
 
-// mustGetProfilePath must get location that the profile will be written to.
-func mustGetProfilePath() string {
-	return filepath.Join(mustGetConfigPath(), globalMinioProfilePath)
-}
-
 func main() {
 	app := registerApp()
 	app.Before = func(c *cli.Context) error {

namespace-lock.go

@@ -29,7 +29,7 @@ type nsParam struct {
 
 // nsLock - provides primitives for locking critical namespace regions.
 type nsLock struct {
-	*sync.RWMutex
+	sync.RWMutex
 	ref uint
 }
 
@@ -37,7 +37,7 @@ type nsLock struct {
 // Unlock, RLock and RUnlock.
 type nsLockMap struct {
 	lockMap map[nsParam]*nsLock
-	mutex   *sync.Mutex
+	mutex   sync.Mutex
 }
 
 // Global name space lock.
@@ -47,7 +47,6 @@ var nsMutex *nsLockMap
 func initNSLock() {
 	nsMutex = &nsLockMap{
 		lockMap: make(map[nsParam]*nsLock),
-		mutex:   &sync.Mutex{},
 	}
 }
 
@@ -59,8 +58,7 @@ func (n *nsLockMap) lock(volume, path string, readLock bool) {
 	nsLk, found := n.lockMap[param]
 	if !found {
 		nsLk = &nsLock{
-			RWMutex: &sync.RWMutex{},
-			ref:     0,
+			ref: 0,
 		}
 		n.lockMap[param] = nsLk
 	}

namespace-lock_test.go

@@ -0,0 +1,123 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "testing"
+
+// Tests functionality provided by namespace lock.
+func TestNamespaceLockTest(t *testing.T) {
+	// List of test cases.
+	testCases := []struct {
+		lk               func(s1, s2 string)
+		unlk             func(s1, s2 string)
+		rlk              func(s1, s2 string)
+		runlk            func(s1, s2 string)
+		lkCount          int
+		lockedRefCount   uint
+		unlockedRefCount uint
+		shouldPass       bool
+	}{
+		{
+			lk:               nsMutex.Lock,
+			unlk:             nsMutex.Unlock,
+			lockedRefCount:   1,
+			unlockedRefCount: 0,
+			shouldPass:       true,
+		},
+		{
+			rlk:              nsMutex.RLock,
+			runlk:            nsMutex.RUnlock,
+			lockedRefCount:   4,
+			unlockedRefCount: 2,
+			shouldPass:       true,
+		},
+		{
+			rlk:              nsMutex.RLock,
+			runlk:            nsMutex.RUnlock,
+			lockedRefCount:   1,
+			unlockedRefCount: 0,
+			shouldPass:       true,
+		},
+	}
+
+	// Run all test cases.
+
+	// Write lock tests.
+	testCase := testCases[0]
+	testCase.lk("a", "b") // lock once.
+	nsLk, ok := nsMutex.lockMap[nsParam{"a", "b"}]
+	if !ok && testCase.shouldPass {
+		t.Errorf("Lock in map missing.")
+	}
+	// Validate loced ref count.
+	if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.lockedRefCount, nsLk.ref)
+	}
+	testCase.unlk("a", "b") // unlock once.
+	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.unlockedRefCount, nsLk.ref)
+	}
+	_, ok = nsMutex.lockMap[nsParam{"a", "b"}]
+	if ok && !testCase.shouldPass {
+		t.Errorf("Lock map found after unlock.")
+	}
+
+	// Read lock tests.
+	testCase = testCases[1]
+	testCase.rlk("a", "b") // lock once.
+	testCase.rlk("a", "b") // lock second time.
+	testCase.rlk("a", "b") // lock third time.
+	testCase.rlk("a", "b") // lock fourth time.
+	nsLk, ok = nsMutex.lockMap[nsParam{"a", "b"}]
+	if !ok && testCase.shouldPass {
+		t.Errorf("Lock in map missing.")
+	}
+	// Validate loced ref count.
+	if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.lockedRefCount, nsLk.ref)
+	}
+	testCase.runlk("a", "b") // unlock once.
+	testCase.runlk("a", "b") // unlock second time.
+	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 2, testCase.unlockedRefCount, nsLk.ref)
+	}
+	_, ok = nsMutex.lockMap[nsParam{"a", "b"}]
+	if !ok && testCase.shouldPass {
+		t.Errorf("Lock map not found.")
+	}
+
+	// Read lock 0 ref count.
+	testCase = testCases[2]
+	testCase.rlk("a", "c") // lock once.
+
+	nsLk, ok = nsMutex.lockMap[nsParam{"a", "c"}]
+	if !ok && testCase.shouldPass {
+		t.Errorf("Lock in map missing.")
+	}
+	// Validate loced ref count.
+	if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 3, testCase.lockedRefCount, nsLk.ref)
+	}
+	testCase.runlk("a", "c") // unlock once.
+	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 3, testCase.unlockedRefCount, nsLk.ref)
+	}
+	_, ok = nsMutex.lockMap[nsParam{"a", "c"}]
+	if ok && !testCase.shouldPass {
+		t.Errorf("Lock map not found.")
+	}
+}

notifier.go

@@ -21,12 +21,12 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/cheggaaa/pb"
 	"github.com/fatih/color"
-	"github.com/olekukonko/ts"
 )
 
 // colorizeUpdateMessage - inspired from Yeoman project npm package https://github.com/yeoman/update-notifier
-func colorizeUpdateMessage(updateString string) (string, error) {
+func colorizeUpdateMessage(updateString string) string {
 	// Initialize coloring.
 	cyan := color.New(color.FgCyan, color.Bold).SprintFunc()
 	yellow := color.New(color.FgYellow, color.Bold).SprintfFunc()
@@ -44,14 +44,17 @@ func colorizeUpdateMessage(updateString string) (string, error) {
 	maxContentWidth := line1Length
 	line1Rest := maxContentWidth - line1Length
 
-	terminal, err := ts.GetSize()
-	if err != nil {
-		return "", err
+	// termWidth is set to a default one to use when we are
+	// not able to calculate terminal width via OS syscalls
+	termWidth := 25
+
+	if width, err := pb.GetTerminalWidth(); err == nil {
+		termWidth = width
 	}
 
 	var message string
 	switch {
-	case len(line1Str) > terminal.Col():
+	case len(line1Str) > termWidth:
 		message = "\n" + line1InColor + "\n"
 	default:
 		// On windows terminal turn off unicode characters.
@@ -75,5 +78,5 @@ func colorizeUpdateMessage(updateString string) (string, error) {
 			bottom + "\n"
 	}
 	// Return the final message.
-	return message, nil
+	return message
 }

object-api-getobject_test.go

@@ -0,0 +1,424 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+)
+
+// Wrapper for calling GetObject tests for both XL multiple disks and single node setup.
+func TestGetObject(t *testing.T) {
+	ExecObjectLayerTest(t, testGetObject)
+}
+
+// ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated.
+func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// Setup for the tests.
+	bucketName := getRandomBucketName()
+	objectName := "test-object"
+	// create bucket.
+	err := obj.MakeBucket(bucketName)
+	// Stop the test if creation of the bucket fails.
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// set of byte data for PutObject.
+	// object has to be inserted before running tests for GetObject.
+	// this is required even to assert the GetObject data,
+	// since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical.
+	bytesData := []struct {
+		byteData []byte
+	}{
+		{generateBytesData(6 * 1024 * 1024)},
+	}
+	// set of inputs for uploading the objects before tests for downloading is done.
+	putObjectInputs := []struct {
+		bucketName    string
+		objectName    string
+		contentLength int64
+		textData      []byte
+		metaData      map[string]string
+	}{
+		// case - 1.
+		{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
+	}
+	// iterate through the above set of inputs and upkoad the object.
+	for i, input := range putObjectInputs {
+		// uploading the object.
+		_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
+		// if object upload fails stop the test.
+		if err != nil {
+			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
+		}
+	}
+	// set of empty buffers used to fill GetObject data.
+	buffers := []*bytes.Buffer{
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+	}
+
+	// test cases with set of inputs
+	testCases := []struct {
+		bucketName  string
+		objectName  string
+		startOffset int64
+		length      int64
+		// data obtained/fetched from GetObject.
+		getObjectData *bytes.Buffer
+		// writer which governs the write into the `getObjectData`.
+		writer io.Writer
+		// flag indicating whether the test for given ase should pass.
+		shouldPass bool
+		// expected Result.
+		expectedData []byte
+		err          error
+	}{
+		// Test case 1-4.
+		// Cases with invalid bucket names.
+		{".test", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: .test")},
+		{"------", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: ------")},
+		{"$this-is-not-valid-too", "obj", 0, 0, nil, nil, false,
+			[]byte(""), fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")},
+		{"a", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: a")},
+		// Test case - 5.
+		// Case with invalid object names.
+		{bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"#")},
+		// Test case - 6.
+		// 	Valid object and bucket names but non-existent bucket.
+		//	{"abc", "def", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket not found: abc")},
+		// A custom writer is sent as an argument.
+		// Its designed to return a EOF error after reading `n` bytes, where `n` is the argument when initializing the EOF writer.
+		// This is to simulate the case of cache not filling up completly, since the EOFWriter doesn't allow the write to complete,
+		// the cache gets filled up with partial data. The following up test case will read the object completly, tests the
+		// purging of the cache during the incomplete write.
+		//	Test case - 7.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[0], NewEOFWriter(buffers[0], 100), false, []byte{}, io.EOF},
+		// Test case with start offset set to 0 and length set to size of the object.
+		// Fetching the entire object.
+		// 	Test case - 8.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], true, bytesData[0].byteData, nil},
+		// Test case with content-range 1 to objectSize .
+		// Test case - 9.
+		{bucketName, objectName, 1, int64(len(bytesData[0].byteData) - 1), buffers[1], buffers[1], true, bytesData[0].byteData[1:], nil},
+		// Test case with content-range 100 to objectSize - 100.
+		// Test case - 10.
+		{bucketName, objectName, 100, int64(len(bytesData[0].byteData) - 200), buffers[1], buffers[1], true,
+			bytesData[0].byteData[100 : len(bytesData[0].byteData)-100], nil},
+		// Test case with offset greater than the size of the object
+		// Test case - 11.
+		{bucketName, objectName, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), buffers[0],
+			NewEOFWriter(buffers[0], 100), false, []byte{},
+			InvalidRange{int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}},
+		// Test case with offset greater than the size of the object.
+		// Test case - 12.
+		{bucketName, objectName, -1, int64(len(bytesData[0].byteData)), buffers[0], new(bytes.Buffer), false, []byte{}, errUnexpected},
+		// Test case length parameter is more than the object size.
+		// Test case - 13.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData) + 1), buffers[1], buffers[1], false, bytesData[0].byteData,
+			InvalidRange{0, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData))}},
+		// Test case with `length` parameter set to a negative value.
+		// Test case - 14.
+		{bucketName, objectName, 0, int64(-1), buffers[1], buffers[1], false, bytesData[0].byteData, errUnexpected},
+		// Test case with offset + length > objectSize parameter set to a negative value.
+		// Test case - 15.
+		{bucketName, objectName, 2, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], false, bytesData[0].byteData,
+			InvalidRange{2, int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}},
+		// Test case with the writer set to nil.
+		// Test case - 16.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], nil, false, bytesData[0].byteData, errUnexpected},
+	}
+
+	for i, testCase := range testCases {
+		err = obj.GetObject(testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead.", i+1, instanceType, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if !strings.Contains(err.Error(), testCase.err.Error()) {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.err.Error(), err.Error())
+			}
+		}
+		// Since there are cases for which GetObject fails, this is
+		// necessary. Test passes as expected, but the output values
+		// are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if !bytes.Equal(testCase.expectedData, testCase.getObjectData.Bytes()) {
+				t.Errorf("Test %d: %s: Data Mismatch: Expected data and the fetched data from GetObject doesn't match.", i+1, instanceType)
+			}
+			// empty the buffer so that it can be used to further cases.
+			testCase.getObjectData.Reset()
+		}
+	}
+}
+
+// Benchmarks for ObjectLayer.GetObject().
+// The intent is to benchamrk GetObject for various sizes ranging from few bytes to 100MB.
+// Also each of these Benchmarks are run both XL and FS backends.
+
+// BenchmarkGetObjectVerySmallFS - Benchmark FS.GetObject() for object size of 10 bytes.
+func BenchmarkGetObjectVerySmallFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(10))
+}
+
+// BenchmarkGetObjectVerySmallXL - Benchmark XL.GetObject() for object size of 10 bytes.
+func BenchmarkGetObjectVerySmallXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(10))
+}
+
+// BenchmarkGetObject10KbFS - Benchmark FS.GetObject() for object size of 10KB.
+func BenchmarkGetObject10KbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(10*1024))
+}
+
+// BenchmarkGetObject10KbXL - Benchmark XL.GetObject() for object size of 10KB.
+func BenchmarkGetObject10KbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(10*1024))
+}
+
+// BenchmarkGetObject100KbFS - Benchmark FS.GetObject() for object size of 100KB.
+func BenchmarkGetObject100KbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(100*1024))
+}
+
+// BenchmarkGetObject100KbXL - Benchmark XL.GetObject() for object size of 100KB.
+func BenchmarkGetObject100KbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(100*1024))
+}
+
+// BenchmarkGetObject1MbFS - Benchmark FS.GetObject() for object size of 1MB.
+func BenchmarkGetObject1MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(1024*1024))
+}
+
+// BenchmarkGetObject1MbXL - Benchmark XL.GetObject() for object size of 1MB.
+func BenchmarkGetObject1MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(1024*1024))
+}
+
+// BenchmarkGetObject5MbFS - Benchmark FS.GetObject() for object size of 5MB.
+func BenchmarkGetObject5MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(5*1024*1024))
+}
+
+// BenchmarkGetObject5MbXL - Benchmark XL.GetObject() for object size of 5MB.
+func BenchmarkGetObject5MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(5*1024*1024))
+}
+
+// BenchmarkGetObject10MbFS - Benchmark FS.GetObject() for object size of 10MB.
+func BenchmarkGetObject10MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(10*1024*1024))
+}
+
+// BenchmarkGetObject10MbXL - Benchmark XL.GetObject() for object size of 10MB.
+func BenchmarkGetObject10MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(10*1024*1024))
+}
+
+// BenchmarkGetObject25MbFS - Benchmark FS.GetObject() for object size of 25MB.
+func BenchmarkGetObject25MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(25*1024*1024))
+
+}
+
+// BenchmarkGetObject25MbXL - Benchmark XL.GetObject() for object size of 25MB.
+func BenchmarkGetObject25MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(25*1024*1024))
+}
+
+// BenchmarkGetObject50MbFS - Benchmark FS.GetObject() for object size of 50MB.
+func BenchmarkGetObject50MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(50*1024*1024))
+}
+
+// BenchmarkGetObject50MbXL - Benchmark XL.GetObject() for object size of 50MB.
+func BenchmarkGetObject50MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(50*1024*1024))
+}
+
+// BenchmarkGetObject100MbFS - Benchmark FS.GetObject() for object size of 100MB.
+func BenchmarkGetObject100MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(100*1024*1024))
+}
+
+// BenchmarkGetObject100MbXL - Benchmark XL.GetObject() for object size of 100MB.
+func BenchmarkGetObject100MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(100*1024*1024))
+}
+
+// BenchmarkGetObject200MbFS - Benchmark FS.GetObject() for object size of 200MB.
+func BenchmarkGetObject200MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(200*1024*1024))
+}
+
+// BenchmarkGetObject200MbXL - Benchmark XL.GetObject() for object size of 200MB.
+func BenchmarkGetObject200MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(200*1024*1024))
+}
+
+// BenchmarkGetObject500MbFS - Benchmark FS.GetObject() for object size of 500MB.
+func BenchmarkGetObject500MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(500*1024*1024))
+}
+
+// BenchmarkGetObject500MbXL - Benchmark XL.GetObject() for object size of 500MB.
+func BenchmarkGetObject500MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(500*1024*1024))
+}
+
+// BenchmarkGetObject1GbFS - Benchmark FS.GetObject() for object size of 1GB.
+func BenchmarkGetObject1GbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmark(1024*1024*1024))
+}
+
+// BenchmarkGetObjectGbXL - Benchmark XL.GetObject() for object size of 1GB.
+func BenchmarkGetObject1GbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmark(1024*1024*1024))
+}
+
+// The intent is to benchamrk GetObject for various sizes ranging from few bytes to 100MB.
+// Also each of these BenchmarkParallels are run both XL and FS backends.
+
+// BenchmarkParallelGetObjectVerySmallFS - BenchmarkParallel FS.GetObject() for object size of 10 bytes.
+func BenchmarkParallelGetObjectVerySmallFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(10))
+}
+
+// BenchmarkParallelGetObjectVerySmallXL - BenchmarkParallel XL.GetObject() for object size of 10 bytes.
+func BenchmarkParallelGetObjectVerySmallXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(10))
+}
+
+// BenchmarkParallelGetObject10KbFS - BenchmarkParallel FS.GetObject() for object size of 10KB.
+func BenchmarkParallelGetObject10KbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(10*1024))
+}
+
+// BenchmarkParallelGetObject10KbXL - BenchmarkParallel XL.GetObject() for object size of 10KB.
+func BenchmarkParallelGetObject10KbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(10*1024))
+}
+
+// BenchmarkParallelGetObject100KbFS - BenchmarkParallel FS.GetObject() for object size of 100KB.
+func BenchmarkParallelGetObject100KbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(100*1024))
+}
+
+// BenchmarkParallelGetObject100KbXL - BenchmarkParallel XL.GetObject() for object size of 100KB.
+func BenchmarkParallelGetObject100KbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(100*1024))
+}
+
+// BenchmarkParallelGetObject1MbFS - BenchmarkParallel FS.GetObject() for object size of 1MB.
+func BenchmarkParallelGetObject1MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(1024*1024))
+}
+
+// BenchmarkParallelGetObject1MbXL - BenchmarkParallel XL.GetObject() for object size of 1MB.
+func BenchmarkParallelGetObject1MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(1024*1024))
+}
+
+// BenchmarkParallelGetObject5MbFS - BenchmarkParallel FS.GetObject() for object size of 5MB.
+func BenchmarkParallelGetObject5MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(5*1024*1024))
+}
+
+// BenchmarkParallelGetObject5MbXL - BenchmarkParallel XL.GetObject() for object size of 5MB.
+func BenchmarkParallelGetObject5MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(5*1024*1024))
+}
+
+// BenchmarkParallelGetObject10MbFS - BenchmarkParallel FS.GetObject() for object size of 10MB.
+func BenchmarkParallelGetObject10MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(10*1024*1024))
+}
+
+// BenchmarkParallelGetObject10MbXL - BenchmarkParallel XL.GetObject() for object size of 10MB.
+func BenchmarkParallelGetObject10MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(10*1024*1024))
+}
+
+// BenchmarkParallelGetObject25MbFS - BenchmarkParallel FS.GetObject() for object size of 25MB.
+func BenchmarkParallelGetObject25MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(25*1024*1024))
+
+}
+
+// BenchmarkParallelGetObject25MbXL - BenchmarkParallel XL.GetObject() for object size of 25MB.
+func BenchmarkParallelGetObject25MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(25*1024*1024))
+}
+
+// BenchmarkParallelGetObject50MbFS - BenchmarkParallel FS.GetObject() for object size of 50MB.
+func BenchmarkParallelGetObject50MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(50*1024*1024))
+}
+
+// BenchmarkParallelGetObject50MbXL - BenchmarkParallel XL.GetObject() for object size of 50MB.
+func BenchmarkParallelGetObject50MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(50*1024*1024))
+}
+
+// BenchmarkParallelGetObject100MbFS - BenchmarkParallel FS.GetObject() for object size of 100MB.
+func BenchmarkParallelGetObject100MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(100*1024*1024))
+}
+
+// BenchmarkParallelGetObject100MbXL - BenchmarkParallel XL.GetObject() for object size of 100MB.
+func BenchmarkParallelGetObject100MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(100*1024*1024))
+}
+
+// BenchmarkParallelGetObject200MbFS - BenchmarkParallel FS.GetObject() for object size of 200MB.
+func BenchmarkParallelGetObject200MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(200*1024*1024))
+}
+
+// BenchmarkParallelGetObject200MbXL - BenchmarkParallel XL.GetObject() for object size of 200MB.
+func BenchmarkParallelGetObject200MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(200*1024*1024))
+}
+
+// BenchmarkParallelGetObject500MbFS - BenchmarkParallel FS.GetObject() for object size of 500MB.
+func BenchmarkParallelGetObject500MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(500*1024*1024))
+}
+
+// BenchmarkParallelGetObject500MbXL - BenchmarkParallel XL.GetObject() for object size of 500MB.
+func BenchmarkParallelGetObject500MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(500*1024*1024))
+}
+
+// BenchmarkParallelGetObject1GbFS - BenchmarkParallel FS.GetObject() for object size of 1GB.
+func BenchmarkParallelGetObject1GbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", returnGetObjectBenchmarkParallel(1024*1024*1024))
+}
+
+// BenchmarkParallelGetObjectGbXL - BenchmarkParallel XL.GetObject() for object size of 1GB.
+func BenchmarkParallelGetObject1GbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", returnGetObjectBenchmarkParallel(1024*1024*1024))
+}

object-api-getobjectinfo_test.go

@@ -18,11 +18,6 @@ package main
 
 import (
 	"bytes"
-	"crypto/md5"
-	"encoding/hex"
-	"io/ioutil"
-	"os"
-	"strconv"
 	"testing"
 )
 
@@ -32,7 +27,7 @@ func TestGetObjectInfo(t *testing.T) {
 }
 
 // Testing GetObjectInfo().
-func testGetObjectInfo(obj ObjectLayer, instanceType string, t *testing.T) {
+func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// This bucket is used for testing getObjectInfo operations.
 	err := obj.MakeBucket("test-getobjectinfo")
 	if err != nil {
@@ -109,48 +104,3 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t *testing.T) {
 		}
 	}
 }
-
-func BenchmarkGetObjectFS(b *testing.B) {
-	// Make a temporary directory to use as the obj.
-	directory, err := ioutil.TempDir("", "minio-benchmark-getobject")
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer os.RemoveAll(directory)
-
-	// Create the obj.
-	obj, err := newFSObjects(directory)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Make a bucket and put in a few objects.
-	err = obj.MakeBucket("bucket")
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	text := "Jack and Jill went up the hill / To fetch a pail of water."
-	hasher := md5.New()
-	hasher.Write([]byte(text))
-	metadata := make(map[string]string)
-	for i := 0; i < 10; i++ {
-		metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
-		_, err = obj.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		var buffer = new(bytes.Buffer)
-		err = obj.GetObject("bucket", "object"+strconv.Itoa(i%10), 0, int64(len([]byte(text))), buffer)
-		if err != nil {
-			b.Error(err)
-		}
-		if buffer.Len() != len(text) {
-			b.Errorf("GetObject returned incorrect length %d (should be %d)\n", buffer.Len(), len(text))
-		}
-	}
-}

object-api-listobjects_test.go

@@ -32,7 +32,7 @@ func TestListObjects(t *testing.T) {
 }
 
 // Unit test for ListObjects in general.
-func testListObjects(obj ObjectLayer, instanceType string, t *testing.T) {
+func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 	// This bucket is used for testing ListObject operations.
 	err := obj.MakeBucket("test-bucket-list-object")
@@ -577,7 +577,7 @@ func BenchmarkListObjects(b *testing.B) {
 	if err != nil {
 		b.Fatal(err)
 	}
-	defer os.RemoveAll(directory)
+	defer removeAll(directory)
 
 	// Create the obj.
 	obj, err := newFSObjects(directory)

object-api-multipart_test.go

@@ -18,6 +18,8 @@ package main
 
 import (
 	"bytes"
+	"crypto/md5"
+	"encoding/hex"
 	"fmt"
 	"strings"
 	"testing"
@@ -29,7 +31,7 @@ func TestObjectNewMultipartUpload(t *testing.T) {
 }
 
 // Tests validate creation of new multipart upload instance.
-func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t *testing.T) {
+func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 	bucket := "minio-bucket"
 	object := "minio-object"
@@ -73,7 +75,7 @@ func TestObjectAPIIsUploadIDExists(t *testing.T) {
 }
 
 // Tests validates the validator for existence of uploadID.
-func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t *testing.T) {
+func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	bucket := "minio-bucket"
 	object := "minio-object"
 
@@ -97,13 +99,93 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t *test
 	}
 }
 
+// Wrapper for calling TestPutObjectPartDiskNotFound tests for both XL
+// write quorum.
+func TestPutObjectPartDiskNotFound(t *testing.T) {
+	ExecObjectLayerDiskNotFoundTest(t, testPutObjectPartDiskNotFound)
+}
+
+// testPutObjectPartDiskNotFound - Tests validate PutObjectPart behavior when disks go offline.
+func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
+	bucketNames := []string{"minio-bucket", "minio-2-bucket"}
+	objectNames := []string{"minio-object-1.txt"}
+	uploadIDs := []string{}
+
+	// bucketnames[0].
+	// objectNames[0].
+	// uploadIds [0].
+	// Create bucket before intiating NewMultipartUpload.
+	err := obj.MakeBucket(bucketNames[0])
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Initiate Multipart Upload on the above created bucket.
+	uploadID, err := obj.NewMultipartUpload(bucketNames[0], objectNames[0], nil)
+	if err != nil {
+		// Failed to create NewMultipartUpload, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Remove some random disk.
+	for _, disk := range disks[:7] {
+		removeAll(disk)
+	}
+
+	uploadIDs = append(uploadIDs, uploadID)
+
+	// Create multipart parts.
+	// Need parts to be uploaded before MultipartLists can be called and tested.
+	createPartCases := []struct {
+		bucketName      string
+		objName         string
+		uploadID        string
+		PartID          int
+		inputReaderData string
+		inputMd5        string
+		intputDataSize  int64
+		expectedMd5     string
+	}{
+		// Case 1-5.
+		// Creating sequence of parts for same uploadID.
+		// Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID.
+		{bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("ijkl")), "09a0877d04abf8759f99adec02baf579"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("mnop")), "e132e96a5ddad6da8b07bba6f6131fef"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 5, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("mnop")), "e132e96a5ddad6da8b07bba6f6131fef"},
+	}
+	// Iterating over creatPartCases to generate multipart chunks.
+	for _, testCase := range createPartCases {
+		_, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+	}
+
+	// This causes quorum failure verify.
+	removeAll(disks[len(disks)-1])
+
+	// Object part upload should fail with quorum not available.
+	testCase := createPartCases[len(createPartCases)-1]
+	_, err = obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
+	if err == nil {
+		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
+	}
+	expectedErr := InsufficientWriteQuorum{}
+	if err.Error() != expectedErr.Error() {
+		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr, err)
+	}
+}
+
 // Wrapper for calling PutObjectPart tests for both XL multiple disks and single node setup.
 func TestObjectAPIPutObjectPart(t *testing.T) {
 	ExecObjectLayerTest(t, testObjectAPIPutObjectPart)
 }
 
 // Tests validate correctness of PutObjectPart.
-func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t *testing.T) {
+func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// Generating cases for which the PutObjectPart fails.
 	bucket := "minio-bucket"
 	object := "minio-object"
@@ -127,7 +209,9 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t *testing
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
-	failCases := []struct {
+	// Collection of non-exhaustive PutObjectPart test cases, valid errors
+	// and success responses.
+	testCases := []struct {
 		bucketName      string
 		objName         string
 		uploadID        string
@@ -185,7 +269,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t *testing
 		// Test case - 14.
 		// Input with size less than the size of actual data inside the reader.
 		{bucket, object, uploadID, 1, "abcd", "a35", int64(len("abcd") - 1), false, "",
-			fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated e2fc714c4727ee9395f324cd2e7f331f")},
+			fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated 900150983cd24fb0d6963f7d28e17f72")},
 		// Test case - 15-18.
 		// Validating for success cases.
 		{bucket, object, uploadID, 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), true, "", nil},
@@ -194,11 +278,10 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t *testing
 		{bucket, object, uploadID, 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), true, "", nil},
 	}
 
-	for i, testCase := range failCases {
-		actualMd5Hex, actualErr := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize,
-			bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
+	// Validate all the test cases.
+	for i, testCase := range testCases {
+		actualMd5Hex, actualErr := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
 		// All are test cases above are expected to fail.
-
 		if actualErr != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
 		}
@@ -208,8 +291,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t *testing
 		// Failed as expected, but does it fail for the expected reason.
 		if actualErr != nil && !testCase.shouldPass {
 			if testCase.expectedError.Error() != actualErr.Error() {
-				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1,
-					instanceType, testCase.expectedError.Error(), actualErr.Error())
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error())
 			}
 		}
 		// Test passes as expected, but the output values are verified for correctness here.
@@ -228,7 +310,7 @@ func TestListMultipartUploads(t *testing.T) {
 }
 
 // testListMultipartUploads - Tests validate listing of multipart uploads.
-func testListMultipartUploads(obj ObjectLayer, instanceType string, t *testing.T) {
+func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 	bucketNames := []string{"minio-bucket", "minio-2-bucket", "minio-3-bucket"}
 	objectNames := []string{"minio-object-1.txt", "minio-object.txt", "neymar-1.jpeg", "neymar.jpeg", "parrot-1.png", "parrot.png"}
@@ -237,7 +319,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t *testing.T
 	// bucketnames[0].
 	// objectNames[0].
 	// uploadIds [0].
-	// Create bucket before intiating NewMultipartUpload.
+	// Create bucket before initiating NewMultipartUpload.
 	err := obj.MakeBucket(bucketNames[0])
 	if err != nil {
 		// Failed to create newbucket, abort.
@@ -285,7 +367,8 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t *testing.T
 	// Initiate Multipart Upload on bucketNames[2].
 	//  Used to test the listing for the case of multiple objects for a given bucket.
 	for i := 0; i < 6; i++ {
-		uploadID, err := obj.NewMultipartUpload(bucketNames[2], objectNames[i], nil)
+		var uploadID string
+		uploadID, err = obj.NewMultipartUpload(bucketNames[2], objectNames[i], nil)
 		if err != nil {
 			// Failed to create NewMultipartUpload, abort.
 			t.Fatalf("%s : %s", instanceType, err.Error())
@@ -330,8 +413,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t *testing.T
 	}
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize,
-			bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
+		_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -931,15 +1013,17 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t *testing.T
 		},
 	}
 
+	// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
+	// and success responses.
 	testCases := []struct {
-		// Inputs to ListObjects.
+		// Inputs to ListMultipartUploads.
 		bucket         string
 		prefix         string
 		keyMarker      string
 		uploadIDMarker string
 		delimiter      string
 		maxUploads     int
-		// Expected output of ListObjects.
+		// Expected output of ListMultipartUploads.
 		expectedResult ListMultipartsInfo
 		expectedErr    error
 		// Flag indicating whether the test is expected to pass or not.
@@ -1121,13 +1205,13 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t *testing.T
 	}
 }
 
-// Wrapper for calling TestListObjectParts tests for both XL multiple disks and single node setup.
-func TestListObjectParts(t *testing.T) {
-	ExecObjectLayerTest(t, testListObjectParts)
+// Wrapper for calling TestListObjectPartsDiskNotFound tests for both XL multiple disks and single node setup.
+func TestListObjectPartsDiskNotFound(t *testing.T) {
+	ExecObjectLayerDiskNotFoundTest(t, testListObjectPartsDiskNotFound)
 }
 
-// testListMultipartUploads - Tests validate listing of multipart uploads.
-func testListObjectParts(obj ObjectLayer, instanceType string, t *testing.T) {
+// testListObjectParts - Tests validate listing of object parts when disks go offline.
+func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
 
 	bucketNames := []string{"minio-bucket", "minio-2-bucket"}
 	objectNames := []string{"minio-object-1.txt"}
@@ -1149,6 +1233,9 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t *testing.T) {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
+	// Remove some random disk.
+	removeDiskN(disks, 1)
+
 	uploadIDs = append(uploadIDs, uploadID)
 
 	// Create multipart parts.
@@ -1173,13 +1260,15 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t *testing.T) {
 	}
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize,
-			bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
+		_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
 	}
 
+	// Remove one disk.
+	removeDiskN(disks, 1)
+
 	partInfos := []ListPartsInfo{
 		// partinfos - 0.
 		{
@@ -1253,6 +1342,8 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t *testing.T) {
 		},
 	}
 
+	// Collection of non-exhaustive ListObjectParts test cases, valid errors
+	// and success responses.
 	testCases := []struct {
 		bucket           string
 		object           string
@@ -1355,3 +1446,483 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t *testing.T) {
 		}
 	}
 }
+
+// Wrapper for calling TestListObjectParts tests for both XL multiple disks and single node setup.
+func TestListObjectParts(t *testing.T) {
+	ExecObjectLayerTest(t, testListObjectParts)
+}
+
+// testListMultipartUploads - Tests validate listing of multipart uploads.
+func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler) {
+
+	bucketNames := []string{"minio-bucket", "minio-2-bucket"}
+	objectNames := []string{"minio-object-1.txt"}
+	uploadIDs := []string{}
+
+	// bucketnames[0].
+	// objectNames[0].
+	// uploadIds [0].
+	// Create bucket before intiating NewMultipartUpload.
+	err := obj.MakeBucket(bucketNames[0])
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+	// Initiate Multipart Upload on the above created bucket.
+	uploadID, err := obj.NewMultipartUpload(bucketNames[0], objectNames[0], nil)
+	if err != nil {
+		// Failed to create NewMultipartUpload, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	uploadIDs = append(uploadIDs, uploadID)
+
+	// Create multipart parts.
+	// Need parts to be uploaded before MultipartLists can be called and tested.
+	createPartCases := []struct {
+		bucketName      string
+		objName         string
+		uploadID        string
+		PartID          int
+		inputReaderData string
+		inputMd5        string
+		intputDataSize  int64
+		expectedMd5     string
+	}{
+		// Case 1-4.
+		// Creating sequence of parts for same uploadID.
+		// Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID.
+		{bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"},
+	}
+	// Iterating over creatPartCases to generate multipart chunks.
+	for _, testCase := range createPartCases {
+		_, err := obj.PutObjectPart(testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMd5)
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+	}
+
+	partInfos := []ListPartsInfo{
+		// partinfos - 0.
+		{
+			Bucket:   bucketNames[0],
+			Object:   objectNames[0],
+			MaxParts: 10,
+			UploadID: uploadIDs[0],
+			Parts: []partInfo{
+				{
+					PartNumber: 1,
+					Size:       4,
+					ETag:       "e2fc714c4727ee9395f324cd2e7f331f",
+				},
+				{
+					PartNumber: 2,
+					Size:       4,
+					ETag:       "1f7690ebdd9b4caf8fab49ca1757bf27",
+				},
+				{
+					PartNumber: 3,
+					Size:       4,
+					ETag:       "09a0877d04abf8759f99adec02baf579",
+				},
+				{
+					PartNumber: 4,
+					Size:       4,
+					ETag:       "e132e96a5ddad6da8b07bba6f6131fef",
+				},
+			},
+		},
+		// partinfos - 1.
+		{
+			Bucket:               bucketNames[0],
+			Object:               objectNames[0],
+			MaxParts:             3,
+			NextPartNumberMarker: 3,
+			IsTruncated:          true,
+			UploadID:             uploadIDs[0],
+			Parts: []partInfo{
+				{
+					PartNumber: 1,
+					Size:       4,
+					ETag:       "e2fc714c4727ee9395f324cd2e7f331f",
+				},
+				{
+					PartNumber: 2,
+					Size:       4,
+					ETag:       "1f7690ebdd9b4caf8fab49ca1757bf27",
+				},
+				{
+					PartNumber: 3,
+					Size:       4,
+					ETag:       "09a0877d04abf8759f99adec02baf579",
+				},
+			},
+		},
+		// partinfos - 2.
+		{
+			Bucket:      bucketNames[0],
+			Object:      objectNames[0],
+			MaxParts:    2,
+			IsTruncated: false,
+			UploadID:    uploadIDs[0],
+			Parts: []partInfo{
+				{
+					PartNumber: 4,
+					Size:       4,
+					ETag:       "e132e96a5ddad6da8b07bba6f6131fef",
+				},
+			},
+		},
+	}
+
+	// Collection of non-exhaustive ListObjectParts test cases, valid errors
+	// and success responses.
+	testCases := []struct {
+		bucket           string
+		object           string
+		uploadID         string
+		partNumberMarker int
+		maxParts         int
+		// Expected output of ListPartsInfo.
+		expectedResult ListPartsInfo
+		expectedErr    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names (Test number 1-4).
+		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Test cases for listing uploadID with single part.
+		// Valid bucket names, but they donot exist (Test number 5-7).
+		{"volatile-bucket-1", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		// Test case for Asserting for invalid objectName (Test number 8).
+		{bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false},
+		// Asserting for Invalid UploadID (Test number 9).
+		{bucketNames[0], objectNames[0], "abc", 0, 0, ListPartsInfo{}, InvalidUploadID{UploadID: "abc"}, false},
+		// Test case for uploadID with multiple parts (Test number 12).
+		{bucketNames[0], objectNames[0], uploadIDs[0], 0, 10, partInfos[0], nil, true},
+		// Test case with maxParts set to less than number of parts (Test number 13).
+		{bucketNames[0], objectNames[0], uploadIDs[0], 0, 3, partInfos[1], nil, true},
+		// Test case with partNumberMarker set (Test number 14)-.
+		{bucketNames[0], objectNames[0], uploadIDs[0], 3, 2, partInfos[2], nil, true},
+	}
+
+	for i, testCase := range testCases {
+		actualResult, actualErr := obj.ListObjectParts(testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, actualErr.Error())
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr.Error(), actualErr.Error())
+			}
+		}
+		// Passes as expected, but asserting the results.
+		if actualErr == nil && testCase.shouldPass {
+			expectedResult := testCase.expectedResult
+			// Asserting the MaxParts.
+			if actualResult.MaxParts != expectedResult.MaxParts {
+				t.Errorf("Test %d: %s: Expected the MaxParts to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxParts, actualResult.MaxParts)
+			}
+			// Asserting Object Name.
+			if actualResult.Object != expectedResult.Object {
+				t.Errorf("Test %d: %s: Expected Object name to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Object, actualResult.Object)
+			}
+			// Asserting UploadID.
+			if actualResult.UploadID != expectedResult.UploadID {
+				t.Errorf("Test %d: %s: Expected UploadID to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.UploadID, actualResult.UploadID)
+			}
+			// Asserting NextPartNumberMarker.
+			if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker {
+				t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker)
+			}
+			// Asserting PartNumberMarker.
+			if actualResult.PartNumberMarker != expectedResult.PartNumberMarker {
+				t.Errorf("Test %d: %s: Expected PartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.PartNumberMarker, actualResult.PartNumberMarker)
+			}
+			// Asserting the BucketName.
+			if actualResult.Bucket != expectedResult.Bucket {
+				t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket)
+			}
+			// Asserting IsTruncated.
+			if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
+				t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
+			}
+			// Asserting the number of Parts.
+			if len(expectedResult.Parts) != len(actualResult.Parts) {
+				t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts))
+			} else {
+				// Iterating over the partInfos and asserting the fields.
+				for j, actualMetaData := range actualResult.Parts {
+					//  Asserting the PartNumber in the PartInfo.
+					if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber {
+						t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber)
+					}
+					//  Asserting the Size in the PartInfo.
+					if actualMetaData.Size != expectedResult.Parts[j].Size {
+						t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size)
+					}
+					//  Asserting the ETag in the PartInfo.
+					if actualMetaData.ETag != expectedResult.Parts[j].ETag {
+						t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag)
+					}
+				}
+			}
+		}
+	}
+}
+
+// Test for validating complete Multipart upload.
+func TestObjectCompleteMultipartUpload(t *testing.T) {
+	ExecObjectLayerTest(t, testObjectCompleteMultipartUpload)
+}
+
+// Tests validate CompleteMultipart functionality.
+func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// Calculates MD5 sum of the given byte array.
+	findMD5 := func(toBeHashed []byte) string {
+		hasher := md5.New()
+		hasher.Write(toBeHashed)
+		return hex.EncodeToString(hasher.Sum(nil))
+	}
+	var err error
+	var uploadID string
+	bucketNames := []string{"minio-bucket", "minio-2-bucket"}
+	objectNames := []string{"minio-object-1.txt"}
+	uploadIDs := []string{}
+
+	// bucketnames[0].
+	// objectNames[0].
+	// uploadIds [0].
+	// Create bucket before intiating NewMultipartUpload.
+	err = obj.MakeBucket(bucketNames[0])
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+	// Initiate Multipart Upload on the above created bucket.
+	uploadID, err = obj.NewMultipartUpload(bucketNames[0], objectNames[0], nil)
+	if err != nil {
+		// Failed to create NewMultipartUpload, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+
+	uploadIDs = append(uploadIDs, uploadID)
+	// Parts with size greater than 5 MB.
+	// Generating a 6MB byte array.
+	validPart := bytes.Repeat([]byte("abcdef"), 1024*1024)
+	validPartMD5 := findMD5(validPart)
+	// Create multipart parts.
+	// Need parts to be uploaded before CompleteMultiPartUpload can be called tested.
+	parts := []struct {
+		bucketName      string
+		objName         string
+		uploadID        string
+		PartID          int
+		inputReaderData string
+		inputMd5        string
+		intputDataSize  int64
+	}{
+		// Case 1-4.
+		// Creating sequence of parts for same uploadID.
+		{bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd"))},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh"))},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd"))},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd"))},
+		// Part with size larger than 5Mb.
+		{bucketNames[0], objectNames[0], uploadIDs[0], 5, string(validPart), validPartMD5, int64(len(string(validPart)))},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 6, string(validPart), validPartMD5, int64(len(string(validPart)))},
+	}
+	// Iterating over creatPartCases to generate multipart chunks.
+	for _, part := range parts {
+		_, err = obj.PutObjectPart(part.bucketName, part.objName, part.uploadID, part.PartID, part.intputDataSize, bytes.NewBufferString(part.inputReaderData), part.inputMd5)
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err)
+		}
+	}
+	// Parts to be sent as input for CompleteMultipartUpload.
+	inputParts := []struct {
+		parts []completePart
+	}{
+		// inputParts - 0.
+		// Case for replicating ETag mismatch.
+		{
+			[]completePart{
+				{ETag: "abcd", PartNumber: 1},
+			},
+		},
+		// inputParts - 1.
+		// should error out with part too small.
+		{
+			[]completePart{
+				{ETag: "e2fc714c4727ee9395f324cd2e7f331f", PartNumber: 1},
+				{ETag: "1f7690ebdd9b4caf8fab49ca1757bf27", PartNumber: 2},
+			},
+		},
+		// inputParts - 2.
+		// Case with invalid Part number.
+		{
+			[]completePart{
+				{ETag: "e2fc714c4727ee9395f324cd2e7f331f", PartNumber: 10},
+			},
+		},
+		// inputParts - 3.
+		// Case with valid part.
+		// Part size greater than 5MB.
+		{
+			[]completePart{
+				{ETag: validPartMD5, PartNumber: 5},
+			},
+		},
+		// inputParts - 4.
+		// Used to verify that the other remaining parts are deleted after
+		// a successful call to CompleteMultipartUpload.
+		{
+			[]completePart{
+				{ETag: validPartMD5, PartNumber: 6},
+			},
+		},
+	}
+	s3MD5, err := completeMultipartMD5(inputParts[3].parts...)
+	if err != nil {
+		t.Fatalf("Obtaining S3MD5 failed")
+	}
+
+	// Test cases with sample input values for CompleteMultipartUpload.
+	testCases := []struct {
+		bucket   string
+		object   string
+		uploadID string
+		parts    []completePart
+		// Expected output of CompleteMultipartUpload.
+		expectedS3MD5 string
+		expectedErr   error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names (Test number 1-4).
+		{".test", "", "", []completePart{}, "", BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", []completePart{}, "", BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", []completePart{}, "", BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", []completePart{}, "", BucketNameInvalid{Bucket: "ad"}, false},
+		// Test cases for listing uploadID with single part.
+		// Valid bucket names, but they donot exist (Test number 5-7).
+		{"volatile-bucket-1", "", "", []completePart{}, "", BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "", "", []completePart{}, "", BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "", "", []completePart{}, "", BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		// Test case for Asserting for invalid objectName (Test number 8).
+		{bucketNames[0], "", "", []completePart{}, "", ObjectNameInvalid{Bucket: bucketNames[0]}, false},
+		// Asserting for Invalid UploadID (Test number 9).
+		{bucketNames[0], objectNames[0], "abc", []completePart{}, "", InvalidUploadID{UploadID: "abc"}, false},
+		// Test case with invalid Part Etag (Test number 10-11).
+		{bucketNames[0], objectNames[0], uploadIDs[0], []completePart{{ETag: "abc"}}, "", fmt.Errorf("encoding/hex: odd length hex string"), false},
+		{bucketNames[0], objectNames[0], uploadIDs[0], []completePart{{ETag: "abcz"}}, "", fmt.Errorf("encoding/hex: invalid byte: U+007A 'z'"), false},
+		// Part number 0 doesn't exist, expecting InvalidPart error (Test number 12).
+		{bucketNames[0], objectNames[0], uploadIDs[0], []completePart{{ETag: "abcd", PartNumber: 0}}, "", InvalidPart{}, false},
+		// // Upload and PartNumber exists, But a deliberate ETag mismatch is introduced (Test number 13).
+		{bucketNames[0], objectNames[0], uploadIDs[0], inputParts[0].parts, "", BadDigest{}, false},
+		// Test case with non existent object name (Test number 14).
+		{bucketNames[0], "my-object", uploadIDs[0], []completePart{{ETag: "abcd", PartNumber: 1}}, "", InvalidUploadID{UploadID: uploadIDs[0]}, false},
+		// Testing for Part being too small (Test number 15).
+		{bucketNames[0], objectNames[0], uploadIDs[0], inputParts[1].parts, "", PartTooSmall{PartNumber: 1}, false},
+		// TestCase with invalid Part Number (Test number 16).
+		// Should error with Invalid Part .
+		{bucketNames[0], objectNames[0], uploadIDs[0], inputParts[2].parts, "", InvalidPart{}, false},
+		// Test case with unsorted parts (Test number 17).
+		{bucketNames[0], objectNames[0], uploadIDs[0], inputParts[3].parts, s3MD5, nil, true},
+		// The other parts will be flushed after a successful completePart (Test number 18).
+		// the case above successfully completes CompleteMultipartUpload, the remaining Parts will be flushed.
+		// Expecting to fail with Invalid UploadID.
+		{bucketNames[0], objectNames[0], uploadIDs[0], inputParts[4].parts, "", InvalidUploadID{UploadID: uploadIDs[0]}, false},
+	}
+
+	for i, testCase := range testCases {
+		actualResult, actualErr := obj.CompleteMultipartUpload(testCase.bucket, testCase.object, testCase.uploadID, testCase.parts)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, actualErr)
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, instanceType, testCase.expectedErr, actualErr)
+			}
+		}
+		// Passes as expected, but asserting the results.
+		if actualErr == nil && testCase.shouldPass {
+
+			// Asserting IsTruncated.
+			if actualResult != testCase.expectedS3MD5 {
+				t.Errorf("Test %d: %s: Expected the result to be \"%v\", but found it to \"%v\"", i+1, instanceType, testCase.expectedS3MD5, actualResult)
+			}
+		}
+	}
+}
+
+// Benchmarks for ObjectLayer.PutObjectPart().
+// The intent is to benchamrk PutObjectPart for various sizes ranging from few bytes to 100MB.
+// Also each of these Benchmarks are run both XL and FS backends.
+
+// BenchmarkPutObjectPart5MbFS - Benchmark FS.PutObjectPart() for object size of 5MB.
+func BenchmarkPutObjectPart5MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(5*1024*1024))
+}
+
+// BenchmarkPutObjectPart5MbXL - Benchmark XL.PutObjectPart() for object size of 5MB.
+func BenchmarkPutObjectPart5MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(5*1024*1024))
+}
+
+// BenchmarkPutObjectPart10MbFS - Benchmark FS.PutObjectPart() for object size of 10MB.
+func BenchmarkPutObjectPart10MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(10*1024*1024))
+}
+
+// BenchmarkPutObjectPart10MbXL - Benchmark XL.PutObjectPart() for object size of 10MB.
+func BenchmarkPutObjectPart10MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(10*1024*1024))
+}
+
+// BenchmarkPutObjectPart25MbFS - Benchmark FS.PutObjectPart() for object size of 25MB.
+func BenchmarkPutObjectPart25MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(25*1024*1024))
+
+}
+
+// BenchmarkPutObjectPart25MbXL - Benchmark XL.PutObjectPart() for object size of 25MB.
+func BenchmarkPutObjectPart25MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(25*1024*1024))
+}
+
+// BenchmarkPutObjectPart50MbFS - Benchmark FS.PutObjectPart() for object size of 50MB.
+func BenchmarkPutObjectPart50MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(50*1024*1024))
+}
+
+// BenchmarkPutObjectPart50MbXL - Benchmark XL.PutObjectPart() for object size of 50MB.
+func BenchmarkPutObjectPart50MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(50*1024*1024))
+}
+
+// BenchmarkPutObjectPart100MbFS - Benchmark FS.PutObjectPart() for object size of 100MB.
+func BenchmarkPutObjectPart100MbFS(b *testing.B) {
+	benchmarkPutObjectPart(b, "FS", returnPutObjectPartBenchmark(100*1024*1024))
+}
+
+// BenchmarkPutObjectPart100MbXL - Benchmark XL.PutObjectPart() for object size of 100MB.
+func BenchmarkPutObjectPart100MbXL(b *testing.B) {
+	benchmarkPutObjectPart(b, "XL", returnPutObjectPartBenchmark(100*1024*1024))
+}

object-api-putobject_test.go

@@ -0,0 +1,591 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+// Wrapper for calling PutObject tests for both XL multiple disks and single node setup.
+func TestObjectAPIPutObject(t *testing.T) {
+	ExecObjectLayerTest(t, testObjectAPIPutObject)
+}
+
+// Tests validate correctness of PutObject.
+func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// Generating cases for which the PutObject fails.
+	bucket := "minio-bucket"
+	object := "minio-object"
+
+	// Create bucket.
+	err := obj.MakeBucket(bucket)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Creating a dummy bucket for tests.
+	err = obj.MakeBucket("unused-bucket")
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	failCases := []struct {
+		bucketName      string
+		objName         string
+		inputReaderData string
+		inputMeta       map[string]string
+		intputDataSize  int64
+		// flag indicating whether the test should pass.
+		shouldPass bool
+		// expected error output.
+		expectedMd5   string
+		expectedError error
+	}{
+		// Test case  1-4.
+		// Cases with invalid bucket name.
+		{".test", "obj", "", nil, 0, false, "", fmt.Errorf("%s", "Bucket name invalid: .test")},
+		{"------", "obj", "", nil, 0, false, "", fmt.Errorf("%s", "Bucket name invalid: ------")},
+		{"$this-is-not-valid-too", "obj", "", nil, 0, false, "",
+			fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")},
+		{"a", "obj", "", nil, 0, false, "", fmt.Errorf("%s", "Bucket name invalid: a")},
+		// Test case - 5.
+		// Case with invalid object names.
+		{bucket, "", "", nil, 0, false, "", fmt.Errorf("%s", "Object name invalid: minio-bucket#")},
+		// Test case - 6.
+		// Valid object and bucket names but non-existent bucket.
+		{"abc", "def", "", nil, 0, false, "", fmt.Errorf("%s", "Bucket not found: abc")},
+		// Test case - 7.
+		// Input to replicate Md5 mismatch.
+		{bucket, object, "", map[string]string{"md5Sum": "a35"}, 0, false, "",
+			fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated "+"d41d8cd98f00b204e9800998ecf8427e")},
+		// Test case - 8.
+		// Input with size more than the size of actual data inside the reader.
+		{bucket, object, "abcd", map[string]string{"md5Sum": "a35"}, int64(len("abcd") + 1), false, "",
+			fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated e2fc714c4727ee9395f324cd2e7f331f")},
+		// Test case - 9.
+		// Input with size less than the size of actual data inside the reader.
+		{bucket, object, "abcd", map[string]string{"md5Sum": "a35"}, int64(len("abcd") - 1), false, "",
+			fmt.Errorf("%s", "Bad digest: Expected a35 is not valid with what we calculated 900150983cd24fb0d6963f7d28e17f72")},
+		// Test case - 10-13.
+		// Validating for success cases.
+		{bucket, object, "abcd", map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), true, "", nil},
+		{bucket, object, "efgh", map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), true, "", nil},
+		{bucket, object, "ijkl", map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), true, "", nil},
+		{bucket, object, "mnop", map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), true, "", nil},
+	}
+
+	for i, testCase := range failCases {
+		actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMeta)
+		// All are test cases above are expected to fail.
+
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if testCase.expectedError.Error() != actualErr.Error() {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error())
+			}
+		}
+		// Test passes as expected, but the output values are verified for correctness here.
+		if actualErr == nil && testCase.shouldPass {
+			// Asserting whether the md5 output is correct.
+			if testCase.inputMeta["md5Sum"] != actualMd5Hex {
+				t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualMd5Hex)
+			}
+		}
+	}
+}
+
+// Wrapper for calling PutObject tests for both XL multiple disks case
+// when quorum is not available.
+func TestObjectAPIPutObjectDiskNotFound(t *testing.T) {
+	ExecObjectLayerDiskNotFoundTest(t, testObjectAPIPutObjectDiskNotFOund)
+}
+
+// Tests validate correctness of PutObject.
+func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
+	// Generating cases for which the PutObject fails.
+	bucket := "minio-bucket"
+	object := "minio-object"
+
+	// Create bucket.
+	err := obj.MakeBucket(bucket)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Creating a dummy bucket for tests.
+	err = obj.MakeBucket("unused-bucket")
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Take 6 disks down, one more we loose quorum on 16 disk node.
+	for _, disk := range disks[:6] {
+		removeAll(disk)
+	}
+
+	testCases := []struct {
+		bucketName      string
+		objName         string
+		inputReaderData string
+		inputMeta       map[string]string
+		intputDataSize  int64
+		// flag indicating whether the test should pass.
+		shouldPass bool
+		// expected error output.
+		expectedMd5   string
+		expectedError error
+	}{
+		// Validating for success cases.
+		{bucket, object, "abcd", map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), true, "", nil},
+		{bucket, object, "efgh", map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), true, "", nil},
+		{bucket, object, "ijkl", map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), true, "", nil},
+		{bucket, object, "mnop", map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), true, "", nil},
+	}
+
+	for i, testCase := range testCases {
+		actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMeta)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if testCase.expectedError.Error() != actualErr.Error() {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1,
+					instanceType, testCase.expectedError.Error(), actualErr.Error())
+			}
+		}
+		// Test passes as expected, but the output values are verified for correctness here.
+		if actualErr == nil && testCase.shouldPass {
+			// Asserting whether the md5 output is correct.
+			if testCase.inputMeta["md5Sum"] != actualMd5Hex {
+				t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualMd5Hex)
+			}
+		}
+	}
+
+	// This causes quorum failure verify.
+	removeAll(disks[len(disks)-1])
+
+	// Validate the last test.
+	testCase := struct {
+		bucketName      string
+		objName         string
+		inputReaderData string
+		inputMeta       map[string]string
+		intputDataSize  int64
+		// flag indicating whether the test should pass.
+		shouldPass bool
+		// expected error output.
+		expectedMd5   string
+		expectedError error
+	}{
+		bucket,
+		object,
+		"mnop",
+		map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"},
+		int64(len("mnop")),
+		false,
+		"",
+		InsufficientWriteQuorum{},
+	}
+	_, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewBufferString(testCase.inputReaderData), testCase.inputMeta)
+	if actualErr != nil && testCase.shouldPass {
+		t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", len(testCases)+1, instanceType, actualErr.Error())
+	}
+	// Failed as expected, but does it fail for the expected reason.
+	if actualErr != nil && !testCase.shouldPass {
+		if testCase.expectedError.Error() != actualErr.Error() {
+			t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", len(testCases)+1, instanceType, testCase.expectedError.Error(), actualErr.Error())
+		}
+	}
+}
+
+// Wrapper for calling PutObject tests for both XL multiple disks and single node setup.
+func TestObjectAPIPutObjectStaleFiles(t *testing.T) {
+	ExecObjectLayerStaleFilesTest(t, testObjectAPIPutObjectStaleFiles)
+}
+
+// Tests validate correctness of PutObject.
+func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
+	// Generating cases for which the PutObject fails.
+	bucket := "minio-bucket"
+	object := "minio-object"
+
+	// Create bucket.
+	err := obj.MakeBucket(bucket)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	data := []byte("hello, world")
+	// Create object.
+	_, err = obj.PutObject(bucket, object, int64(len(data)), bytes.NewReader(data), nil)
+	if err != nil {
+		// Failed to create object, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	for _, disk := range disks {
+		tmpMetaDir := path.Join(disk, minioMetaBucket, tmpMetaPrefix)
+		if !isDirEmpty(tmpMetaDir) {
+			t.Fatalf("%s: expected: empty, got: non-empty", tmpMetaDir)
+		}
+	}
+}
+
+// Wrapper for calling Multipart PutObject tests for both XL multiple disks and single node setup.
+func TestObjectAPIMultipartPutObjectStaleFiles(t *testing.T) {
+	ExecObjectLayerStaleFilesTest(t, testObjectAPIMultipartPutObjectStaleFiles)
+}
+
+// Tests validate correctness of PutObject.
+func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
+	// Generating cases for which the PutObject fails.
+	bucket := "minio-bucket"
+	object := "minio-object"
+
+	// Create bucket.
+	err := obj.MakeBucket(bucket)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Initiate Multipart Upload on the above created bucket.
+	uploadID, err := obj.NewMultipartUpload(bucket, object, nil)
+	if err != nil {
+		// Failed to create NewMultipartUpload, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Upload part1.
+	fiveMBBytes := bytes.Repeat([]byte("a"), 5*1024*1024)
+	md5Writer := md5.New()
+	md5Writer.Write(fiveMBBytes)
+	etag1 := hex.EncodeToString(md5Writer.Sum(nil))
+	_, err = obj.PutObjectPart(bucket, object, uploadID, 1, int64(len(fiveMBBytes)), bytes.NewReader(fiveMBBytes), etag1)
+	if err != nil {
+		// Failed to upload object part, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Upload part2.
+	data := []byte("hello, world")
+	md5Writer = md5.New()
+	md5Writer.Write(data)
+	etag2 := hex.EncodeToString(md5Writer.Sum(nil))
+	_, err = obj.PutObjectPart(bucket, object, uploadID, 2, int64(len(data)), bytes.NewReader(data), etag2)
+	if err != nil {
+		// Failed to upload object part, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Complete multipart.
+	parts := []completePart{
+		{ETag: etag1, PartNumber: 1},
+		{ETag: etag2, PartNumber: 2},
+	}
+	_, err = obj.CompleteMultipartUpload(bucket, object, uploadID, parts)
+	if err != nil {
+		// Failed to complete multipart upload, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	for _, disk := range disks {
+		tmpMetaDir := path.Join(disk, minioMetaBucket, tmpMetaPrefix)
+		files, err := ioutil.ReadDir(tmpMetaDir)
+		if err != nil {
+			// Its OK to have non-existen tmpMetaDir.
+			if os.IsNotExist(err) {
+				continue
+			}
+
+			// Print the error
+			t.Errorf("%s", err)
+		}
+
+		if len(files) != 0 {
+			t.Fatalf("%s: expected: empty, got: non-empty. content: %s", tmpMetaDir, files)
+		}
+	}
+}
+
+// Benchmarks for ObjectLayer.PutObject().
+// The intent is to benchamrk PutObject for various sizes ranging from few bytes to 100MB.
+// Also each of these Benchmarks are run both XL and FS backends.
+
+// BenchmarkPutObjectVerySmallFS - Benchmark FS.PutObject() for object size of 10 bytes.
+func BenchmarkPutObjectVerySmallFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10))
+}
+
+// BenchmarkPutObjectVerySmallXL - Benchmark XL.PutObject() for object size of 10 bytes.
+func BenchmarkPutObjectVerySmallXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10))
+}
+
+// BenchmarkPutObject10KbFS - Benchmark FS.PutObject() for object size of 10KB.
+func BenchmarkPutObject10KbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10*1024))
+}
+
+// BenchmarkPutObject10KbXL - Benchmark XL.PutObject() for object size of 10KB.
+func BenchmarkPutObject10KbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10*1024))
+}
+
+// BenchmarkPutObject100KbFS - Benchmark FS.PutObject() for object size of 100KB.
+func BenchmarkPutObject100KbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(100*1024))
+}
+
+// BenchmarkPutObject100KbXL - Benchmark XL.PutObject() for object size of 100KB.
+func BenchmarkPutObject100KbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(100*1024))
+}
+
+// BenchmarkPutObject1MbFS - Benchmark FS.PutObject() for object size of 1MB.
+func BenchmarkPutObject1MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(1024*1024))
+}
+
+// BenchmarkPutObject1MbXL - Benchmark XL.PutObject() for object size of 1MB.
+func BenchmarkPutObject1MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024))
+}
+
+// BenchmarkPutObject5MbFS - Benchmark FS.PutObject() for object size of 5MB.
+func BenchmarkPutObject5MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(5*1024*1024))
+}
+
+// BenchmarkPutObject5MbXL - Benchmark XL.PutObject() for object size of 5MB.
+func BenchmarkPutObject5MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(5*1024*1024))
+}
+
+// BenchmarkPutObject10MbFS - Benchmark FS.PutObject() for object size of 10MB.
+func BenchmarkPutObject10MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(10*1024*1024))
+}
+
+// BenchmarkPutObject10MbXL - Benchmark XL.PutObject() for object size of 10MB.
+func BenchmarkPutObject10MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(10*1024*1024))
+}
+
+// BenchmarkPutObject25MbFS - Benchmark FS.PutObject() for object size of 25MB.
+func BenchmarkPutObject25MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(25*1024*1024))
+
+}
+
+// BenchmarkPutObject25MbXL - Benchmark XL.PutObject() for object size of 25MB.
+func BenchmarkPutObject25MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(25*1024*1024))
+}
+
+// BenchmarkPutObject50MbFS - Benchmark FS.PutObject() for object size of 50MB.
+func BenchmarkPutObject50MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(50*1024*1024))
+}
+
+// BenchmarkPutObject50MbXL - Benchmark XL.PutObject() for object size of 50MB.
+func BenchmarkPutObject50MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(50*1024*1024))
+}
+
+// BenchmarkPutObject100MbFS - Benchmark FS.PutObject() for object size of 100MB.
+func BenchmarkPutObject100MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(100*1024*1024))
+}
+
+// BenchmarkPutObject100MbXL - Benchmark XL.PutObject() for object size of 100MB.
+func BenchmarkPutObject100MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(100*1024*1024))
+}
+
+// BenchmarkPutObject200MbFS - Benchmark FS.PutObject() for object size of 200MB.
+func BenchmarkPutObject200MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(200*1024*1024))
+}
+
+// BenchmarkPutObject200MbXL - Benchmark XL.PutObject() for object size of 200MB.
+func BenchmarkPutObject200MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(200*1024*1024))
+}
+
+// BenchmarkPutObject500MbFS - Benchmark FS.PutObject() for object size of 500MB.
+func BenchmarkPutObject500MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(500*1024*1024))
+}
+
+// BenchmarkPutObject500MbXL - Benchmark XL.PutObject() for object size of 500MB.
+func BenchmarkPutObject500MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(500*1024*1024))
+}
+
+// BenchmarkPutObject1GbFS - Benchmark FS.PutObject() for object size of 1GB.
+func BenchmarkPutObject1GbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmark(1024*1024*1024))
+}
+
+// BenchmarkPutObjectGbXL - Benchmark XL.PutObject() for object size of 1GB.
+func BenchmarkPutObject1GbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmark(1024*1024*1024))
+}
+
+func BenchmarkParallelPutObjectVerySmallFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(10))
+}
+
+// BenchmarkParallelPutObjectVerySmallXL - BenchmarkParallel XL.PutObject() for object size of 10 bytes.
+func BenchmarkParallelPutObjectVerySmallXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(10))
+}
+
+// BenchmarkParallelPutObject10KbFS - BenchmarkParallel FS.PutObject() for object size of 10KB.
+func BenchmarkParallelPutObject10KbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(10*1024))
+}
+
+// BenchmarkParallelPutObject10KbXL - BenchmarkParallel XL.PutObject() for object size of 10KB.
+func BenchmarkParallelPutObject10KbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(10*1024))
+}
+
+// BenchmarkParallelPutObject100KbFS - BenchmarkParallel FS.PutObject() for object size of 100KB.
+func BenchmarkParallelPutObject100KbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(100*1024))
+}
+
+// BenchmarkParallelPutObject100KbXL - BenchmarkParallel XL.PutObject() for object size of 100KB.
+func BenchmarkParallelPutObject100KbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(100*1024))
+}
+
+// BenchmarkParallelPutObject1MbFS - BenchmarkParallel FS.PutObject() for object size of 1MB.
+func BenchmarkParallelPutObject1MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(1024*1024))
+}
+
+// BenchmarkParallelPutObject1MbXL - BenchmarkParallel XL.PutObject() for object size of 1MB.
+func BenchmarkParallelPutObject1MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(1024*1024))
+}
+
+// BenchmarkParallelPutObject5MbFS - BenchmarkParallel FS.PutObject() for object size of 5MB.
+func BenchmarkParallelPutObject5MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(5*1024*1024))
+}
+
+// BenchmarkParallelPutObject5MbXL - BenchmarkParallel XL.PutObject() for object size of 5MB.
+func BenchmarkParallelPutObject5MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(5*1024*1024))
+}
+
+// BenchmarkParallelPutObject10MbFS - BenchmarkParallel FS.PutObject() for object size of 10MB.
+func BenchmarkParallelPutObject10MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(10*1024*1024))
+}
+
+// BenchmarkParallelPutObject10MbXL - BenchmarkParallel XL.PutObject() for object size of 10MB.
+func BenchmarkParallelPutObject10MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(10*1024*1024))
+}
+
+// BenchmarkParallelPutObject25MbFS - BenchmarkParallel FS.PutObject() for object size of 25MB.
+func BenchmarkParallelPutObject25MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(25*1024*1024))
+
+}
+
+// BenchmarkParallelPutObject25MbXL - BenchmarkParallel XL.PutObject() for object size of 25MB.
+func BenchmarkParallelPutObject25MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(25*1024*1024))
+}
+
+// BenchmarkParallelPutObject50MbFS - BenchmarkParallel FS.PutObject() for object size of 50MB.
+func BenchmarkParallelPutObject50MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(50*1024*1024))
+}
+
+// BenchmarkParallelPutObject50MbXL - BenchmarkParallel XL.PutObject() for object size of 50MB.
+func BenchmarkParallelPutObject50MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(50*1024*1024))
+}
+
+// BenchmarkParallelPutObject100MbFS - BenchmarkParallel FS.PutObject() for object size of 100MB.
+func BenchmarkParallelPutObject100MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(100*1024*1024))
+}
+
+// BenchmarkParallelPutObject100MbXL - BenchmarkParallel XL.PutObject() for object size of 100MB.
+func BenchmarkParallelPutObject100MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(100*1024*1024))
+}
+
+// BenchmarkParallelPutObject200MbFS - BenchmarkParallel FS.PutObject() for object size of 200MB.
+func BenchmarkParallelPutObject200MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(200*1024*1024))
+}
+
+// BenchmarkParallelPutObject200MbXL - BenchmarkParallel XL.PutObject() for object size of 200MB.
+func BenchmarkParallelPutObject200MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(200*1024*1024))
+}
+
+// BenchmarkParallelPutObject500MbFS - BenchmarkParallel FS.PutObject() for object size of 500MB.
+func BenchmarkParallelPutObject500MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(500*1024*1024))
+}
+
+// BenchmarkParallelPutObject500MbXL - BenchmarkParallel XL.PutObject() for object size of 500MB.
+func BenchmarkParallelPutObject500MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(500*1024*1024))
+}
+
+// BenchmarkParallelPutObject1GbFS - BenchmarkParallel FS.PutObject() for object size of 1GB.
+func BenchmarkParallelPutObject1GbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", returnPutObjectBenchmarkParallel(1024*1024*1024))
+}
+
+// BenchmarkParallelPutObjectGbXL - BenchmarkParallel XL.PutObject() for object size of 1GB.
+func BenchmarkParallelPutObject1GbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", returnPutObjectBenchmarkParallel(1024*1024*1024))
+}

object-api_test.go

@@ -1,74 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"io/ioutil"
-	"os"
-
-	. "gopkg.in/check.v1"
-)
-
-type MySuite struct{}
-
-var _ = Suite(&MySuite{})
-
-func (s *MySuite) TestFSAPISuite(c *C) {
-	var storageList []string
-
-	// Initialize name space lock.
-	initNSLock()
-
-	create := func() ObjectLayer {
-		path, err := ioutil.TempDir(os.TempDir(), "minio-")
-		c.Check(err, IsNil)
-		objAPI, err := newFSObjects(path)
-		c.Check(err, IsNil)
-		storageList = append(storageList, path)
-		return objAPI
-	}
-	APITestSuite(c, create)
-	defer removeRoots(c, storageList)
-}
-
-func (s *MySuite) TestXLAPISuite(c *C) {
-	var storageList []string
-
-	// Initialize name space lock.
-	initNSLock()
-
-	create := func() ObjectLayer {
-		var nDisks = 16 // Maximum disks.
-		var erasureDisks []string
-		for i := 0; i < nDisks; i++ {
-			path, err := ioutil.TempDir(os.TempDir(), "minio-")
-			c.Check(err, IsNil)
-			erasureDisks = append(erasureDisks, path)
-		}
-		objAPI, err := newXLObjects(erasureDisks)
-		c.Check(err, IsNil)
-		return objAPI
-	}
-	APITestSuite(c, create)
-	defer removeRoots(c, storageList)
-}
-
-func removeRoots(c *C, roots []string) {
-	for _, root := range roots {
-		os.RemoveAll(root)
-	}
-}

object-common.go

@@ -17,29 +17,48 @@
 package main
 
 import (
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
+	"syscall"
 )
 
 const (
 	// Block size used for all internal operations version 1.
 	blockSizeV1 = 10 * 1024 * 1024 // 10MiB.
+
+	// Staging buffer read size for all internal operations version 1.
+	readSizeV1 = 128 * 1024 // 128KiB.
 )
 
+// Register callback functions that needs to be called when process shutsdown.
+// For now, SIGINT triggers the callbacks, in future controller can trigger
+// shutdown callbacks.
+func registerShutdown(callback func()) {
+	go func() {
+		trapCh := signalTrap(os.Interrupt, syscall.SIGTERM)
+		<-trapCh
+		callback()
+	}()
+}
+
+// isErrIgnored should we ignore this error?, takes a list of errors which can be ignored.
+func isErrIgnored(err error, ignoredErrs []error) bool {
+	for _, ignoredErr := range ignoredErrs {
+		if ignoredErr == err {
+			return true
+		}
+	}
+	return false
+}
+
 // House keeping code needed for FS.
 func fsHouseKeeping(storageDisk StorageAPI) error {
-	// Attempt to create `.minio`.
-	err := storageDisk.MakeVol(minioMetaBucket)
-	if err != nil {
-		if err != errVolumeExists && err != errDiskNotFound {
-			return err
-		}
-	}
 	// Cleanup all temp entries upon start.
-	err = cleanupDir(storageDisk, minioMetaBucket, tmpMetaPrefix)
+	err := cleanupDir(storageDisk, minioMetaBucket, tmpMetaPrefix)
 	if err != nil {
-		return err
+		return toObjectErr(err, minioMetaBucket, tmpMetaPrefix)
 	}
 	return nil
 }
@@ -54,8 +73,8 @@ func newStorageAPI(disk string) (storage StorageAPI, err error) {
 	return newRPCClient(disk)
 }
 
-// House keeping code needed for XL.
-func xlHouseKeeping(storageDisks []StorageAPI) error {
+// Initializes meta volume on all input storage disks.
+func initMetaVolume(storageDisks []StorageAPI) error {
 	// This happens for the first time, but keep this here since this
 	// is the only place where it can be made expensive optimizing all
 	// other calls. Create minio meta volume, if it doesn't exist yet.
@@ -67,7 +86,7 @@ func xlHouseKeeping(storageDisks []StorageAPI) error {
 	// Initialize all disks in parallel.
 	for index, disk := range storageDisks {
 		if disk == nil {
-			errs[index] = errDiskNotFound
+			// Ignore create meta volume on disks which are not found.
 			continue
 		}
 		wg.Add(1)
@@ -77,17 +96,57 @@ func xlHouseKeeping(storageDisks []StorageAPI) error {
 
 			// Attempt to create `.minio`.
 			err := disk.MakeVol(minioMetaBucket)
-			if err != nil && err != errVolumeExists && err != errDiskNotFound {
-				errs[index] = err
-				return
+			if err != nil {
+				switch err {
+				// Ignored errors.
+				case errVolumeExists, errDiskNotFound, errFaultyDisk:
+				default:
+					errs[index] = err
+				}
 			}
+		}(index, disk)
+	}
+
+	// Wait for all cleanup to finish.
+	wg.Wait()
+
+	// Return upon first error.
+	for _, err := range errs {
+		if err == nil {
+			continue
+		}
+		return toObjectErr(err, minioMetaBucket)
+	}
+
+	// Return success here.
+	return nil
+}
+
+// House keeping code needed for XL.
+func xlHouseKeeping(storageDisks []StorageAPI) error {
+	// This happens for the first time, but keep this here since this
+	// is the only place where it can be made expensive optimizing all
+	// other calls. Create metavolume.
+	var wg = &sync.WaitGroup{}
+
+	// Initialize errs to collect errors inside go-routine.
+	var errs = make([]error, len(storageDisks))
+
+	// Initialize all disks in parallel.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int, disk StorageAPI) {
+			// Indicate this wait group is done.
+			defer wg.Done()
+
 			// Cleanup all temp entries upon start.
-			err = cleanupDir(disk, minioMetaBucket, tmpMetaPrefix)
+			err := cleanupDir(disk, minioMetaBucket, tmpMetaPrefix)
 			if err != nil {
 				errs[index] = err
-				return
 			}
-			errs[index] = nil
 		}(index, disk)
 	}
 
@@ -112,21 +171,22 @@ func cleanupDir(storage StorageAPI, volume, dirPath string) error {
 	// Function to delete entries recursively.
 	delFunc = func(entryPath string) error {
 		if !strings.HasSuffix(entryPath, slashSeparator) {
-			// No trailing "/" means that this is a file which can be deleted.
+			// Delete the file entry.
 			return storage.DeleteFile(volume, entryPath)
 		}
+
 		// If it's a directory, list and call delFunc() for each entry.
 		entries, err := storage.ListDir(volume, entryPath)
-		if err != nil {
-			if err == errFileNotFound {
-				// if dirPath prefix never existed.
-				return nil
-			}
+		// If entryPath prefix never existed, safe to ignore.
+		if err == errFileNotFound {
+			return nil
+		} else if err != nil { // For any other errors fail.
 			return err
-		}
+		} // else on success..
+
+		// Recurse and delete all other entries.
 		for _, entry := range entries {
-			err = delFunc(pathJoin(entryPath, entry))
-			if err != nil {
+			if err = delFunc(pathJoin(entryPath, entry)); err != nil {
 				return err
 			}
 		}

object-datatypes.go

@@ -20,96 +20,205 @@ import "time"
 
 // StorageInfo - represents total capacity of underlying storage.
 type StorageInfo struct {
-	Total int64 // Total disk space.
-	Free  int64 // Free total available disk space.
+	// Total disk space.
+	Total int64
+	// Free available disk space.
+	Free int64
 }
 
-// BucketInfo - bucket name and create date
+// BucketInfo - represents bucket metadata.
 type BucketInfo struct {
-	Name    string
+	// Name of the bucket.
+	Name string
+
+	// Date and time when the bucket was created.
 	Created time.Time
 }
 
-// ObjectInfo - object info.
+// ObjectInfo - represents object metadata.
 type ObjectInfo struct {
-	Bucket          string
-	Name            string
-	ModTime         time.Time
-	Size            int64
-	IsDir           bool
-	MD5Sum          string
-	ContentType     string
+	// Name of the bucket.
+	Bucket string
+
+	// Name of the object.
+	Name string
+
+	// Date and time when the object was last modified.
+	ModTime time.Time
+
+	// Total object size.
+	Size int64
+
+	// IsDir indicates if the object is prefix.
+	IsDir bool
+
+	// Hex encoded md5 checksum of the object.
+	MD5Sum string
+
+	// A standard MIME type describing the format of the object.
+	ContentType string
+
+	// Specifies what content encodings have been applied to the object and thus
+	// what decoding mechanisms must be applied to obtain the object referenced
+	// by the Content-Type header field.
 	ContentEncoding string
+
+	// User-Defined metadata
+	UserDefined map[string]string
 }
 
-// ListPartsInfo - various types of object resources.
+// ListPartsInfo - represents list of all parts.
 type ListPartsInfo struct {
-	Bucket               string
-	Object               string
-	UploadID             string
-	StorageClass         string
-	PartNumberMarker     int
-	NextPartNumberMarker int
-	MaxParts             int
-	IsTruncated          bool
+	// Name of the bucket.
+	Bucket string
 
-	Parts        []partInfo
-	EncodingType string
+	// Name of the object.
+	Object string
+
+	// Upload ID identifying the multipart upload whose parts are being listed.
+	UploadID string
+
+	// The class of storage used to store the object.
+	StorageClass string
+
+	// Part number after which listing begins.
+	PartNumberMarker int
+
+	// When a list is truncated, this element specifies the last part in the list,
+	// as well as the value to use for the part-number-marker request parameter
+	// in a subsequent request.
+	NextPartNumberMarker int
+
+	// Maximum number of parts that were allowed in the response.
+	MaxParts int
+
+	// Indicates whether the returned list of parts is truncated.
+	IsTruncated bool
+
+	// List of all parts.
+	Parts []partInfo
+
+	EncodingType string // Not supported yet.
 }
 
-// ListMultipartsInfo - various types of bucket resources for inprogress multipart uploads.
+// ListMultipartsInfo - represnets bucket resources for incomplete multipart uploads.
 type ListMultipartsInfo struct {
-	KeyMarker          string
-	UploadIDMarker     string
-	NextKeyMarker      string
+	// Together with upload-id-marker, this parameter specifies the multipart upload
+	// after which listing should begin.
+	KeyMarker string
+
+	// Together with key-marker, specifies the multipart upload after which listing
+	// should begin. If key-marker is not specified, the upload-id-marker parameter
+	// is ignored.
+	UploadIDMarker string
+
+	// When a list is truncated, this element specifies the value that should be
+	// used for the key-marker request parameter in a subsequent request.
+	NextKeyMarker string
+
+	// When a list is truncated, this element specifies the value that should be
+	// used for the upload-id-marker request parameter in a subsequent request.
 	NextUploadIDMarker string
-	EncodingType       string
-	MaxUploads         int
-	IsTruncated        bool
-	Uploads            []uploadMetadata
-	Prefix             string
-	Delimiter          string
-	CommonPrefixes     []string
+
+	// Maximum number of multipart uploads that could have been included in the
+	// response.
+	MaxUploads int
+
+	// Indicates whether the returned list of multipart uploads is truncated. A
+	// value of true indicates that the list was truncated. The list can be truncated
+	// if the number of multipart uploads exceeds the limit allowed or specified
+	// by max uploads.
+	IsTruncated bool
+
+	// List of all pending uploads.
+	Uploads []uploadMetadata
+
+	// When a prefix is provided in the request, The result contains only keys
+	// starting with the specified prefix.
+	Prefix string
+
+	// A character used to truncate the object prefixes.
+	// NOTE: only supported delimiter is '/'.
+	Delimiter string
+
+	// CommonPrefixes contains all (if there are any) keys between Prefix and the
+	// next occurrence of the string specified by delimiter.
+	CommonPrefixes []string
+
+	EncodingType string // Not supported yet.
 }
 
 // ListObjectsInfo - container for list objects.
 type ListObjectsInfo struct {
+	// Indicates whether the returned list objects response is truncated. A
+	// value of true indicates that the list was truncated. The list can be truncated
+	// if the number of objects exceeds the limit allowed or specified
+	// by max keys.
 	IsTruncated bool
-	NextMarker  string
-	Objects     []ObjectInfo
-	Prefixes    []string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects.
+	//
+	// NOTE: This element is returned only if you have delimiter request parameter
+	// specified.
+	NextMarker string
+
+	// List of objects info for this request.
+	Objects []ObjectInfo
+
+	// List of prefixes for this request.
+	Prefixes []string
 }
 
-// partInfo - various types of individual part resources.
+// partInfo - represents individual part metadata.
 type partInfo struct {
-	PartNumber   int
+	// Part number that identifies the part. This is a positive integer between
+	// 1 and 10,000.
+	PartNumber int
+
+	// Date and time at which the part was uploaded.
 	LastModified time.Time
-	ETag         string
-	Size         int64
+
+	// Entity tag returned when the part was initially uploaded.
+	ETag string
+
+	// Size in bytes of the part.
+	Size int64
 }
 
-// uploadMetadata container capturing metadata on in progress multipart upload in a given bucket
+// uploadMetadata - represents metadata in progress multipart upload.
 type uploadMetadata struct {
-	Object       string
-	UploadID     string
-	StorageClass string
-	Initiated    time.Time
+	// Object name for which the multipart upload was initiated.
+	Object string
+
+	// Unique identifier for this multipart upload.
+	UploadID string
+
+	// Date and time at which the multipart upload was initiated.
+	Initiated time.Time
+
+	StorageClass string // Not supported yet.
 }
 
 // completePart - completed part container.
 type completePart struct {
+	// Part number identifying the part. This is a positive integer between 1 and
+	// 10,000
 	PartNumber int
-	ETag       string
+
+	// Entity tag returned when the part was uploaded.
+	ETag string
 }
 
-// completedParts is a sortable interface for Part slice
+// completedParts - is a collection satisfying sort.Interface.
 type completedParts []completePart
 
 func (a completedParts) Len() int           { return len(a) }
 func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
 
-// completeMultipartUpload container for completing multipart upload
+// completeMultipartUpload - represents input fields for completing multipart upload.
 type completeMultipartUpload struct {
 	Parts []completePart `xml:"Part"`
 }

object-errors.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io"
 )
@@ -197,15 +198,6 @@ func (e ObjectNameInvalid) Error() string {
 	return "Object name invalid: " + e.Bucket + "#" + e.Object
 }
 
-// UnExpectedDataSize - Reader contains more/less data than specified.
-type UnExpectedDataSize struct {
-	Size int
-}
-
-func (e UnExpectedDataSize) Error() string {
-	return fmt.Sprintf("Contains more data than specified size of %d bytes.", e.Size)
-}
-
 // IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header.
 type IncompleteBody GenericError
 
@@ -214,6 +206,20 @@ func (e IncompleteBody) Error() string {
 	return e.Bucket + "#" + e.Object + "has incomplete body"
 }
 
+// errInvalidRange - returned when given range value is not valid.
+var errInvalidRange = errors.New("Invalid range")
+
+// InvalidRange - invalid range typed error.
+type InvalidRange struct {
+	offsetBegin  int64
+	offsetEnd    int64
+	resourceSize int64
+}
+
+func (e InvalidRange) Error() string {
+	return fmt.Sprintf("The requested range \"bytes %d-%d/%d\" is not satisfiable.", e.offsetBegin, e.offsetEnd, e.resourceSize)
+}
+
 /// Multipart related errors.
 
 // MalformedUploadID malformed upload id.
@@ -241,18 +247,13 @@ func (e InvalidPart) Error() string {
 	return "One or more of the specified parts could not be found"
 }
 
-// InvalidPartOrder parts are not ordered as Requested
-type InvalidPartOrder struct {
-	UploadID string
-}
-
-func (e InvalidPartOrder) Error() string {
-	return "Invalid part order sent for " + e.UploadID
-}
-
 // PartTooSmall - error if part size is less than 5MB.
-type PartTooSmall struct{}
+type PartTooSmall struct {
+	PartSize   int64
+	PartNumber int
+	PartETag   string
+}
 
 func (e PartTooSmall) Error() string {
-	return "Part size should be atleast 5MB"
+	return fmt.Sprintf("Part size for %d should be atleast 5MB", e.PartNumber)
 }

object-handlers-common.go

@@ -0,0 +1,212 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Validates the preconditions for CopyObject, returns true if CopyObject operation should not proceed.
+// Preconditions supported are:
+//  x-amz-copy-source-if-modified-since
+//  x-amz-copy-source-if-unmodified-since
+//  x-amz-copy-source-if-match
+//  x-amz-copy-source-if-none-match
+func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
+	// Return false for methods other than GET and HEAD.
+	if r.Method != "PUT" {
+		return false
+	}
+	// If the object doesn't have a modtime (IsZero), or the modtime
+	// is obviously garbage (Unix time == 0), then ignore modtimes
+	// and don't process the If-Modified-Since header.
+	if objInfo.ModTime.IsZero() || objInfo.ModTime.Equal(time.Unix(0, 0)) {
+		return false
+	}
+
+	// Headers to be set of object content is not going to be written to the client.
+	writeHeaders := func() {
+		// set common headers
+		setCommonHeaders(w)
+
+		// set object-related metadata headers
+		w.Header().Set("Last-Modified", objInfo.ModTime.UTC().Format(http.TimeFormat))
+
+		if objInfo.MD5Sum != "" {
+			w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+		}
+	}
+	// x-amz-copy-source-if-modified-since: Return the object only if it has been modified
+	// since the specified time otherwise return 412 (precondition failed).
+	ifModifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-modified-since")
+	if ifModifiedSinceHeader != "" {
+		if !ifModifiedSince(objInfo.ModTime, ifModifiedSinceHeader) {
+			// If the object is not modified since the specified time.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// x-amz-copy-source-if-unmodified-since : Return the object only if it has not been
+	// modified since the specified time, otherwise return a 412 (precondition failed).
+	ifUnmodifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-unmodified-since")
+	if ifUnmodifiedSinceHeader != "" {
+		if ifModifiedSince(objInfo.ModTime, ifUnmodifiedSinceHeader) {
+			// If the object is modified since the specified time.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// x-amz-copy-source-if-match : Return the object only if its entity tag (ETag) is the
+	// same as the one specified; otherwise return a 412 (precondition failed).
+	ifMatchETagHeader := r.Header.Get("x-amz-copy-source-if-match")
+	if ifMatchETagHeader != "" {
+		if !isETagEqual(objInfo.MD5Sum, ifMatchETagHeader) {
+			// If the object ETag does not match with the specified ETag.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// If-None-Match : Return the object only if its entity tag (ETag) is different from the
+	// one specified otherwise, return a 304 (not modified).
+	ifNoneMatchETagHeader := r.Header.Get("x-amz-copy-source-if-none-match")
+	if ifNoneMatchETagHeader != "" {
+		if isETagEqual(objInfo.MD5Sum, ifNoneMatchETagHeader) {
+			// If the object ETag matches with the specified ETag.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+	// Object content should be written to http.ResponseWriter
+	return false
+}
+
+// Validates the preconditions. Returns true if GET/HEAD operation should not proceed.
+// Preconditions supported are:
+//  If-Modified-Since
+//  If-Unmodified-Since
+//  If-Match
+//  If-None-Match
+func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
+	// Return false for methods other than GET and HEAD.
+	if r.Method != "GET" && r.Method != "HEAD" {
+		return false
+	}
+	// If the object doesn't have a modtime (IsZero), or the modtime
+	// is obviously garbage (Unix time == 0), then ignore modtimes
+	// and don't process the If-Modified-Since header.
+	if objInfo.ModTime.IsZero() || objInfo.ModTime.Equal(time.Unix(0, 0)) {
+		return false
+	}
+
+	// Headers to be set of object content is not going to be written to the client.
+	writeHeaders := func() {
+		// set common headers
+		setCommonHeaders(w)
+
+		// set object-related metadata headers
+		w.Header().Set("Last-Modified", objInfo.ModTime.UTC().Format(http.TimeFormat))
+
+		if objInfo.MD5Sum != "" {
+			w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+		}
+	}
+	// If-Modified-Since : Return the object only if it has been modified since the specified time,
+	// otherwise return a 304 (not modified).
+	ifModifiedSinceHeader := r.Header.Get("If-Modified-Since")
+	if ifModifiedSinceHeader != "" {
+		if !ifModifiedSince(objInfo.ModTime, ifModifiedSinceHeader) {
+			// If the object is not modified since the specified time.
+			writeHeaders()
+			w.WriteHeader(http.StatusNotModified)
+			return true
+		}
+	}
+
+	// If-Unmodified-Since : Return the object only if it has not been modified since the specified
+	// time, otherwise return a 412 (precondition failed).
+	ifUnmodifiedSinceHeader := r.Header.Get("If-Unmodified-Since")
+	if ifUnmodifiedSinceHeader != "" {
+		if ifModifiedSince(objInfo.ModTime, ifUnmodifiedSinceHeader) {
+			// If the object is modified since the specified time.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// If-Match : Return the object only if its entity tag (ETag) is the same as the one specified;
+	// otherwise return a 412 (precondition failed).
+	ifMatchETagHeader := r.Header.Get("If-Match")
+	if ifMatchETagHeader != "" {
+		if !isETagEqual(objInfo.MD5Sum, ifMatchETagHeader) {
+			// If the object ETag does not match with the specified ETag.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// If-None-Match : Return the object only if its entity tag (ETag) is different from the
+	// one specified otherwise, return a 304 (not modified).
+	ifNoneMatchETagHeader := r.Header.Get("If-None-Match")
+	if ifNoneMatchETagHeader != "" {
+		if isETagEqual(objInfo.MD5Sum, ifNoneMatchETagHeader) {
+			// If the object ETag matches with the specified ETag.
+			writeHeaders()
+			w.WriteHeader(http.StatusNotModified)
+			return true
+		}
+	}
+	// Object content should be written to http.ResponseWriter
+	return false
+}
+
+// returns true if object was modified after givenTime.
+func ifModifiedSince(objTime time.Time, givenTimeStr string) bool {
+	givenTime, err := time.Parse(http.TimeFormat, givenTimeStr)
+	if err != nil {
+		return true
+	}
+	// The Date-Modified header truncates sub-second precision, so
+	// use mtime < t+1s instead of mtime <= t to check for unmodified.
+	if objTime.After(givenTime.Add(1 * time.Second)) {
+		return true
+	}
+	return false
+}
+
+// canonicalizeETag returns ETag with leading and trailing double-quotes removed,
+// if any present
+func canonicalizeETag(etag string) string {
+	canonicalETag := strings.TrimPrefix(etag, "\"")
+	return strings.TrimSuffix(canonicalETag, "\"")
+}
+
+// isETagEqual return true if the canonical representations of two ETag strings
+// are equal, false otherwise
+func isETagEqual(left, right string) bool {
+	return canonicalizeETag(left) == canonicalizeETag(right)
+}

object-handlers.go

@@ -17,10 +17,8 @@
 package main
 
 import (
-	"crypto/sha256"
 	"encoding/hex"
 	"encoding/xml"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -28,7 +26,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	mux "github.com/gorilla/mux"
@@ -68,6 +65,13 @@ func errAllowableObjectNotFound(bucket string, r *http.Request) APIErrorCode {
 	return ErrNoSuchKey
 }
 
+// Simple way to convert a func to io.Writer type.
+type funcToWriter func([]byte) (int, error)
+
+func (f funcToWriter) Write(p []byte) (int, error) {
+	return f(p)
+}
+
 // GetObjectHandler - GET Object
 // ----------
 // This implementation of the GET operation retrieves object. To use GET,
@@ -107,134 +111,71 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// Verify 'If-Modified-Since' and 'If-Unmodified-Since'.
-	lastModified := objInfo.ModTime
-	if checkLastModified(w, r, lastModified) {
-		return
-	}
-	// Verify 'If-Match' and 'If-None-Match'.
-	if checkETag(w, r) {
-		return
-	}
-
+	// Get request range.
 	var hrange *httpRange
-	hrange, err = getRequestedRange(r.Header.Get("Range"), objInfo.Size)
-	if err != nil {
-		writeErrorResponse(w, r, ErrInvalidRange, r.URL.Path)
-		return
+	rangeHeader := r.Header.Get("Range")
+	if rangeHeader != "" {
+		if hrange, err = parseRequestRange(rangeHeader, objInfo.Size); err != nil {
+			// Handle only errInvalidRange
+			// Ignore other parse error and treat it as regular Get request like Amazon S3.
+			if err == errInvalidRange {
+				writeErrorResponse(w, r, ErrInvalidRange, r.URL.Path)
+				return
+			}
+
+			// log the error.
+			errorIf(err, "Invalid request range")
+		}
+
 	}
 
-	// Set standard object headers.
-	setObjectHeaders(w, objInfo, hrange)
-
-	// Set any additional requested response headers.
-	setGetRespHeaders(w, r.URL.Query())
+	// Validate pre-conditions if any.
+	if checkPreconditions(w, r, objInfo) {
+		return
+	}
 
 	// Get the object.
-	startOffset := hrange.start
-	length := hrange.length
-	if length == 0 {
-		length = objInfo.Size - startOffset
+	startOffset := int64(0)
+	length := objInfo.Size
+	if hrange != nil {
+		startOffset = hrange.offsetBegin
+		length = hrange.getLength()
 	}
-	if err := api.ObjectAPI.GetObject(bucket, object, startOffset, length, w); err != nil {
-		errorIf(err, "Writing to client failed.")
-		// Do not send error response here, client would have already died.
+	// Indicates if any data was written to the http.ResponseWriter
+	dataWritten := false
+	// io.Writer type which keeps track if any data was written.
+	writer := funcToWriter(func(p []byte) (int, error) {
+		if !dataWritten {
+			// Set headers on the first write.
+			// Set standard object headers.
+			setObjectHeaders(w, objInfo, hrange)
+
+			// Set any additional requested response headers.
+			setGetRespHeaders(w, r.URL.Query())
+
+			dataWritten = true
+		}
+		return w.Write(p)
+	})
+	// Reads the object at startOffset and writes to mw.
+	if err := api.ObjectAPI.GetObject(bucket, object, startOffset, length, writer); err != nil {
+		errorIf(err, "Unable to write to client.")
+		if !dataWritten {
+			// Error response only if no data has been written to client yet. i.e if
+			// partial data has already been written before an error
+			// occurred then no point in setting StatusCode and
+			// sending error XML.
+			apiErr := toAPIErrorCode(err)
+			writeErrorResponse(w, r, apiErr, r.URL.Path)
+		}
 		return
 	}
-}
-
-var unixEpochTime = time.Unix(0, 0)
-
-// checkLastModified implements If-Modified-Since and
-// If-Unmodified-Since checks.
-//
-// modtime is the modification time of the resource to be served, or
-// IsZero(). return value is whether this request is now complete.
-func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
-	if modtime.IsZero() || modtime.Equal(unixEpochTime) {
-		// If the object doesn't have a modtime (IsZero), or the modtime
-		// is obviously garbage (Unix time == 0), then ignore modtimes
-		// and don't process the If-Modified-Since header.
-		return false
+	if !dataWritten {
+		// If ObjectAPI.GetObject did not return error and no data has
+		// been written it would mean that it is a 0-byte object.
+		// call wrter.Write(nil) to set appropriate headers.
+		writer.Write(nil)
 	}
-
-	// The Date-Modified header truncates sub-second precision, so
-	// use mtime < t+1s instead of mtime <= t to check for unmodified.
-	if _, ok := r.Header["If-Modified-Since"]; ok {
-		// Return the object only if it has been modified since the
-		// specified time, otherwise return a 304 (not modified).
-		t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since"))
-		if err == nil && modtime.Before(t.Add(1*time.Second)) {
-			h := w.Header()
-			// Remove following headers if already set.
-			delete(h, "Content-Type")
-			delete(h, "Content-Length")
-			delete(h, "Content-Range")
-			w.WriteHeader(http.StatusNotModified)
-			return true
-		}
-	} else if _, ok := r.Header["If-Unmodified-Since"]; ok {
-		// Return the object only if it has not been modified since
-		// the specified time, otherwise return a 412 (precondition failed).
-		t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Unmodified-Since"))
-		if err == nil && modtime.After(t.Add(1*time.Second)) {
-			h := w.Header()
-			// Remove following headers if already set.
-			delete(h, "Content-Type")
-			delete(h, "Content-Length")
-			delete(h, "Content-Range")
-			w.WriteHeader(http.StatusPreconditionFailed)
-			return true
-		}
-	}
-	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
-	return false
-}
-
-// checkETag implements If-None-Match and If-Match checks.
-//
-// The ETag must have been previously set in the ResponseWriter's
-// headers. The return value is whether this request is now considered
-// done.
-func checkETag(w http.ResponseWriter, r *http.Request) bool {
-	etag := w.Header().Get("ETag")
-	// Must know ETag.
-	if etag == "" {
-		return false
-	}
-	if inm := r.Header.Get("If-None-Match"); inm != "" {
-		// Return the object only if its entity tag (ETag) is
-		// different from the one specified; otherwise, return a 304
-		// (not modified).
-		if r.Method != "GET" && r.Method != "HEAD" {
-			return false
-		}
-		if inm == etag || inm == "*" {
-			h := w.Header()
-			// Remove following headers if already set.
-			delete(h, "Content-Type")
-			delete(h, "Content-Length")
-			delete(h, "Content-Range")
-			w.WriteHeader(http.StatusNotModified)
-			return true
-		}
-	} else if im := r.Header.Get("If-Match"); im != "" {
-		// Return the object only if its entity tag (ETag) is the same
-		// as the one specified; otherwise, return a 412 (precondition failed).
-		if r.Method != "GET" && r.Method != "HEAD" {
-			return false
-		}
-		if im != etag {
-			h := w.Header()
-			// Remove following headers if already set.
-			delete(h, "Content-Type")
-			delete(h, "Content-Length")
-			delete(h, "Content-Range")
-			w.WriteHeader(http.StatusPreconditionFailed)
-			return true
-		}
-	}
-	return false
 }
 
 // HeadObjectHandler - HEAD Object
@@ -275,14 +216,8 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Verify 'If-Modified-Since' and 'If-Unmodified-Since'.
-	lastModified := objInfo.ModTime
-	if checkLastModified(w, r, lastModified) {
-		return
-	}
-
-	// Verify 'If-Match' and 'If-None-Match'.
-	if checkETag(w, r) {
+	// Validate pre-conditions if any.
+	if checkPreconditions(w, r, objInfo) {
 		return
 	}
 
@@ -320,11 +255,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	// TODO: Reject requests where body/payload is present, for now we
-	// don't even read it.
+	// TODO: Reject requests where body/payload is present, for now we don't even read it.
 
 	// objectSource
-	objectSource := r.Header.Get("X-Amz-Copy-Source")
+	objectSource, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
+	if err != nil {
+		// Save unescaped string as is.
+		objectSource = r.Header.Get("X-Amz-Copy-Source")
+	}
 
 	// Skip the first element if it is '/', split the rest.
 	if strings.HasPrefix(objectSource, "/") {
@@ -356,18 +294,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponse(w, r, toAPIErrorCode(err), objectSource)
 		return
 	}
-	// Verify before writing.
 
-	// Verify x-amz-copy-source-if-modified-since and
-	// x-amz-copy-source-if-unmodified-since.
-	lastModified := objInfo.ModTime
-	if checkCopySourceLastModified(w, r, lastModified) {
-		return
-	}
-
-	// Verify x-amz-copy-source-if-match and
-	// x-amz-copy-source-if-none-match.
-	if checkCopySourceETag(w, r) {
+	// Verify before x-amz-copy-source preconditions before continuing with CopyObject.
+	if checkCopyObjectPreconditions(w, r, objInfo) {
 		return
 	}
 
@@ -426,98 +355,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	pipeReader.Close()
 }
 
-// checkCopySource implements x-amz-copy-source-if-modified-since and
-// x-amz-copy-source-if-unmodified-since checks.
-//
-// modtime is the modification time of the resource to be served, or
-// IsZero(). return value is whether this request is now complete.
-func checkCopySourceLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
-	if modtime.IsZero() || modtime.Equal(unixEpochTime) {
-		// If the object doesn't have a modtime (IsZero), or the modtime
-		// is obviously garbage (Unix time == 0), then ignore modtimes
-		// and don't process the If-Modified-Since header.
-		return false
-	}
-	// The Date-Modified header truncates sub-second precision, so
-	// use mtime < t+1s instead of mtime <= t to check for unmodified.
-	if _, ok := r.Header["x-amz-copy-source-if-modified-since"]; ok {
-		// Return the object only if it has been modified since the
-		// specified time, otherwise return a 304 error (not modified).
-		t, err := time.Parse(http.TimeFormat, r.Header.Get("x-amz-copy-source-if-modified-since"))
-		if err == nil && modtime.Before(t.Add(1*time.Second)) {
-			h := w.Header()
-			// Remove Content headers if set
-			delete(h, "Content-Type")
-			delete(h, "Content-Length")
-			delete(h, "Content-Range")
-			w.WriteHeader(http.StatusNotModified)
-			return true
-		}
-	} else if _, ok := r.Header["x-amz-copy-source-if-unmodified-since"]; ok {
-		// Return the object only if it has not been modified since the
-		// specified time, otherwise return a 412 error (precondition failed).
-		t, err := time.Parse(http.TimeFormat, r.Header.Get("x-amz-copy-source-if-unmodified-since"))
-		if err == nil && modtime.After(t.Add(1*time.Second)) {
-			h := w.Header()
-			// Remove Content headers if set
-			delete(h, "Content-Type")
-			delete(h, "Content-Length")
-			delete(h, "Content-Range")
-			w.WriteHeader(http.StatusPreconditionFailed)
-			return true
-		}
-	}
-	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
-	return false
-}
-
-// checkCopySourceETag implements x-amz-copy-source-if-match and
-// x-amz-copy-source-if-none-match checks.
-//
-// The ETag must have been previously set in the ResponseWriter's
-// headers. The return value is whether this request is now considered
-// complete.
-func checkCopySourceETag(w http.ResponseWriter, r *http.Request) bool {
-	etag := w.Header().Get("ETag")
-	// Tag must be provided...
-	if etag == "" {
-		return false
-	}
-	if inm := r.Header.Get("x-amz-copy-source-if-none-match"); inm != "" {
-		// Return the object only if its entity tag (ETag) is different
-		// from the one specified; otherwise, return a 304 (not modified).
-		if r.Method != "PUT" {
-			return false
-		}
-		if inm == etag || inm == "*" {
-			h := w.Header()
-			// Remove Content headers if set
-			delete(h, "Content-Type")
-			delete(h, "Content-Length")
-			delete(h, "Content-Range")
-			w.WriteHeader(http.StatusNotModified)
-			return true
-		}
-	} else if inm := r.Header.Get("x-amz-copy-source-if-match"); inm != "" {
-		// Return the object only if its entity tag (ETag) is the same
-		// as the one specified; otherwise, return a 412 (precondition failed).
-		if r.Method != "PUT" {
-			return false
-		}
-		if inm != etag {
-			h := w.Header()
-			// Remove Content headers if set
-			delete(h, "Content-Type")
-			delete(h, "Content-Length")
-			delete(h, "Content-Range")
-			w.WriteHeader(http.StatusPreconditionFailed)
-			return true
-		}
-	}
-	return false
-
-}
-
 // PutObjectHandler - PUT Object
 // ----------
 // This implementation of the PUT operation adds an object to a bucket.
@@ -560,9 +397,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	metadata["content-encoding"] = r.Header.Get("Content-Encoding")
 	for key := range r.Header {
 		cKey := http.CanonicalHeaderKey(key)
-		if strings.HasPrefix(cKey, "x-amz-meta-") {
+		if strings.HasPrefix(cKey, "X-Amz-Meta-") {
 			metadata[cKey] = r.Header.Get(cKey)
-		} else if strings.HasPrefix(key, "x-minio-meta-") {
+		} else if strings.HasPrefix(key, "X-Minio-Meta-") {
 			metadata[cKey] = r.Header.Get(cKey)
 		}
 	}
@@ -582,51 +419,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		// Create anonymous object.
 		md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, metadata)
 	case authTypePresigned, authTypeSigned:
-		// Initialize a pipe for data pipe line.
-		reader, writer := io.Pipe()
-		var wg = &sync.WaitGroup{}
-		// Start writing in a routine.
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			shaWriter := sha256.New()
-			multiWriter := io.MultiWriter(shaWriter, writer)
-			if _, wErr := io.CopyN(multiWriter, r.Body, size); wErr != nil {
-				// Pipe closed.
-				if wErr == io.ErrClosedPipe {
-					return
-				}
-				errorIf(wErr, "Unable to read from HTTP body.")
-				writer.CloseWithError(wErr)
-				return
-			}
-			shaPayload := shaWriter.Sum(nil)
-			validateRegion := true // Validate region.
-			var s3Error APIErrorCode
-			if isRequestSignatureV4(r) {
-				s3Error = doesSignatureMatch(hex.EncodeToString(shaPayload), r, validateRegion)
-			} else if isRequestPresignedSignatureV4(r) {
-				s3Error = doesPresignedSignatureMatch(hex.EncodeToString(shaPayload), r, validateRegion)
-			}
-			var sErr error
-			if s3Error != ErrNone {
-				if s3Error == ErrSignatureDoesNotMatch {
-					sErr = errSignatureMismatch
-				} else {
-					sErr = fmt.Errorf("%v", getAPIError(s3Error))
-				}
-				writer.CloseWithError(sErr)
-				return
-			}
-			writer.Close()
-		}()
-
+		// Initialize signature verifier.
+		reader := newSignVerify(r)
 		// Create object.
 		md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata)
-		// Close the pipe.
-		reader.Close()
-		// Wait for all the routines to finish.
-		wg.Wait()
 	}
 	if err != nil {
 		errorIf(err, "Unable to create an object.")
@@ -737,6 +533,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	}
 
 	var partMD5 string
+	incomingMD5 := hex.EncodeToString(md5Bytes)
 	switch getRequestAuthType(r) {
 	default:
 		// For all unknown auth types return error.
@@ -748,55 +545,12 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
-		// No need to verify signature, anonymous request access is
-		// already allowed.
-		hexMD5 := hex.EncodeToString(md5Bytes)
-		partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, hexMD5)
+		// No need to verify signature, anonymous request access is already allowed.
+		partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5)
 	case authTypePresigned, authTypeSigned:
-		// Initialize a pipe for data pipe line.
-		reader, writer := io.Pipe()
-		var wg = &sync.WaitGroup{}
-		// Start writing in a routine.
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			shaWriter := sha256.New()
-			multiWriter := io.MultiWriter(shaWriter, writer)
-			if _, wErr := io.CopyN(multiWriter, r.Body, size); wErr != nil {
-				// Pipe closed, just ignore it.
-				if wErr == io.ErrClosedPipe {
-					return
-				}
-				errorIf(wErr, "Unable to read from HTTP request body.")
-				writer.CloseWithError(wErr)
-				return
-			}
-			shaPayload := shaWriter.Sum(nil)
-			validateRegion := true // Validate region.
-			var s3Error APIErrorCode
-			if isRequestSignatureV4(r) {
-				s3Error = doesSignatureMatch(hex.EncodeToString(shaPayload), r, validateRegion)
-			} else if isRequestPresignedSignatureV4(r) {
-				s3Error = doesPresignedSignatureMatch(hex.EncodeToString(shaPayload), r, validateRegion)
-			}
-			if s3Error != ErrNone {
-				if s3Error == ErrSignatureDoesNotMatch {
-					err = errSignatureMismatch
-				} else {
-					err = fmt.Errorf("%v", getAPIError(s3Error))
-				}
-				writer.CloseWithError(err)
-				return
-			}
-			// Close the writer.
-			writer.Close()
-		}()
-		md5SumHex := hex.EncodeToString(md5Bytes)
-		partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, md5SumHex)
-		// Close the pipe.
-		reader.Close()
-		// Wait for all the routines to finish.
-		wg.Wait()
+		// Initialize signature verifier.
+		reader := newSignVerify(r)
+		partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5)
 	}
 	if err != nil {
 		errorIf(err, "Unable to create object part.")
@@ -843,6 +597,19 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 	writeSuccessNoContent(w)
 }
 
+// Send whitespace character, once every 5secs, until CompleteMultipartUpload is done.
+// CompleteMultipartUpload method of the object layer indicates that it's done via doneCh
+func sendWhiteSpaceChars(w http.ResponseWriter, doneCh <-chan struct{}) {
+	for {
+		select {
+		case <-time.After(5 * time.Second):
+			w.Write([]byte(" "))
+		case <-doneCh:
+			return
+		}
+	}
+}
+
 // ListObjectPartsHandler - List object parts
 func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
@@ -946,21 +713,53 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		completeParts = append(completeParts, part)
 	}
 	// Complete multipart upload.
-	md5Sum, err = api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts)
+	// Send 200 OK
+	setCommonHeaders(w)
+	w.WriteHeader(http.StatusOK)
+	// Xml headers need to be sent before we possibly send whitespace characters
+	// to the client.
+	_, err = w.Write([]byte(xml.Header))
 	if err != nil {
-		errorIf(err, "Unable to complete multipart upload.")
-		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		errorIf(err, "Unable to write XML header for complete multipart upload")
+		writeErrorResponseNoHeader(w, r, ErrInternalError, r.URL.Path)
 		return
 	}
+
+	doneCh := make(chan struct{})
+	// Signal that completeMultipartUpload is over via doneCh
+	go func(doneCh chan<- struct{}) {
+		md5Sum, err = api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts)
+		doneCh <- struct{}{}
+	}(doneCh)
+
+	sendWhiteSpaceChars(w, doneCh)
+
+	if err != nil {
+		errorIf(err, "Unable to complete multipart upload.")
+		switch oErr := err.(type) {
+		case PartTooSmall:
+			// Write part too small error.
+			writePartSmallErrorResponse(w, r, oErr)
+		default:
+			// Handle all other generic issues.
+			writeErrorResponseNoHeader(w, r, toAPIErrorCode(err), r.URL.Path)
+		}
+		return
+	}
+
 	// Get object location.
 	location := getLocation(r)
 	// Generate complete multipart response.
 	response := generateCompleteMultpartUploadResponse(bucket, object, location, md5Sum)
-	encodedSuccessResponse := encodeResponse(response)
-	// Write headers.
-	setCommonHeaders(w)
+	encodedSuccessResponse, err := xml.Marshal(response)
+	if err != nil {
+		errorIf(err, "Unable to parse CompleteMultipartUpload response")
+		writeErrorResponseNoHeader(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
 	// write success response.
-	writeSuccessResponse(w, encodedSuccessResponse)
+	w.Write(encodedSuccessResponse)
+	w.(http.Flusher).Flush()
 }
 
 /// Delete objectAPIHandlers

object-multipart-common.go

@@ -0,0 +1,166 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"path"
+	"sort"
+	"sync"
+	"time"
+)
+
+// A uploadInfo represents the s3 compatible spec.
+type uploadInfo struct {
+	UploadID  string    `json:"uploadId"`  // UploadID for the active multipart upload.
+	Deleted   bool      `json:"deleted"`   // Currently unused, for future use.
+	Initiated time.Time `json:"initiated"` // Indicates when the uploadID was initiated.
+}
+
+// A uploadsV1 represents `uploads.json` metadata header.
+type uploadsV1 struct {
+	Version string       `json:"version"`   // Version of the current `uploads.json`
+	Format  string       `json:"format"`    // Format of the current `uploads.json`
+	Uploads []uploadInfo `json:"uploadIds"` // Captures all the upload ids for a given object.
+}
+
+// byInitiatedTime is a collection satisfying sort.Interface.
+type byInitiatedTime []uploadInfo
+
+func (t byInitiatedTime) Len() int      { return len(t) }
+func (t byInitiatedTime) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
+func (t byInitiatedTime) Less(i, j int) bool {
+	return t[i].Initiated.Before(t[j].Initiated)
+}
+
+// AddUploadID - adds a new upload id in order of its initiated time.
+func (u *uploadsV1) AddUploadID(uploadID string, initiated time.Time) {
+	u.Uploads = append(u.Uploads, uploadInfo{
+		UploadID:  uploadID,
+		Initiated: initiated,
+	})
+	sort.Sort(byInitiatedTime(u.Uploads))
+}
+
+// Index - returns the index of matching the upload id.
+func (u uploadsV1) Index(uploadID string) int {
+	for i, u := range u.Uploads {
+		if u.UploadID == uploadID {
+			return i
+		}
+	}
+	return -1
+}
+
+// readUploadsJSON - get all the saved uploads JSON.
+func readUploadsJSON(bucket, object string, disk StorageAPI) (uploadIDs uploadsV1, err error) {
+	uploadJSONPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
+	// Reads entire `uploads.json`.
+	buf, err := disk.ReadAll(minioMetaBucket, uploadJSONPath)
+	if err != nil {
+		return uploadsV1{}, err
+	}
+
+	// Decode `uploads.json`.
+	if err = json.Unmarshal(buf, &uploadIDs); err != nil {
+		return uploadsV1{}, err
+	}
+
+	// Success.
+	return uploadIDs, nil
+}
+
+// newUploadsV1 - initialize new uploads v1.
+func newUploadsV1(format string) uploadsV1 {
+	uploadIDs := uploadsV1{}
+	uploadIDs.Version = "1.0.0" // Should follow semantic versioning.
+	uploadIDs.Format = format
+	return uploadIDs
+}
+
+// Wrapper which removes all the uploaded parts.
+func cleanupUploadedParts(bucket, object, uploadID string, storageDisks ...StorageAPI) error {
+	var errs = make([]error, len(storageDisks))
+	var wg = &sync.WaitGroup{}
+
+	// Construct uploadIDPath.
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+
+	// Cleanup uploadID for all disks.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			errs[index] = errDiskNotFound
+			continue
+		}
+		wg.Add(1)
+		// Cleanup each uploadID in a routine.
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			err := cleanupDir(disk, minioMetaBucket, uploadIDPath)
+			if err != nil {
+				errs[index] = err
+				return
+			}
+			errs[index] = nil
+		}(index, disk)
+	}
+
+	// Wait for all the cleanups to finish.
+	wg.Wait()
+
+	// Return first error.
+	for _, err := range errs {
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// listMultipartUploadIDs - list all the upload ids from a marker up to 'count'.
+func listMultipartUploadIDs(bucketName, objectName, uploadIDMarker string, count int, disk StorageAPI) ([]uploadMetadata, bool, error) {
+	var uploads []uploadMetadata
+	// Read `uploads.json`.
+	uploadsJSON, err := readUploadsJSON(bucketName, objectName, disk)
+	if err != nil {
+		return nil, false, err
+	}
+	index := 0
+	if uploadIDMarker != "" {
+		for ; index < len(uploadsJSON.Uploads); index++ {
+			if uploadsJSON.Uploads[index].UploadID == uploadIDMarker {
+				// Skip the uploadID as it would already be listed in previous listing.
+				index++
+				break
+			}
+		}
+	}
+	for index < len(uploadsJSON.Uploads) {
+		uploads = append(uploads, uploadMetadata{
+			Object:    objectName,
+			UploadID:  uploadsJSON.Uploads[index].UploadID,
+			Initiated: uploadsJSON.Uploads[index].Initiated,
+		})
+		count--
+		index++
+		if count == 0 {
+			break
+		}
+	}
+	end := (index == len(uploadsJSON.Uploads))
+	return uploads, end, nil
+}

object-utils.go

@@ -30,7 +30,7 @@ import (
 
 const (
 	// Minio meta bucket.
-	minioMetaBucket = ".minio"
+	minioMetaBucket = ".minio.sys"
 	// Multipart meta prefix.
 	mpartMetaPrefix = "multipart"
 	// Tmp meta prefix.

object_api_suite_test.go

@@ -20,46 +20,86 @@ import (
 	"bytes"
 	"crypto/md5"
 	"encoding/hex"
+	"io"
 	"math/rand"
 	"strconv"
 
-	"gopkg.in/check.v1"
+	. "gopkg.in/check.v1"
 )
 
-// APITestSuite - collection of API tests.
-func APITestSuite(c *check.C, create func() ObjectLayer) {
-	testMakeBucket(c, create)
-	testMultipleObjectCreation(c, create)
-	testPaging(c, create)
-	testObjectOverwriteWorks(c, create)
-	testNonExistantBucketOperations(c, create)
-	testBucketRecreateFails(c, create)
-	testPutObjectInSubdir(c, create)
-	testListBuckets(c, create)
-	testListBucketsOrder(c, create)
-	testListObjectsTestsForNonExistantBucket(c, create)
-	testNonExistantObjectInBucket(c, create)
-	testGetDirectoryReturnsObjectNotFound(c, create)
-	testDefaultContentType(c, create)
-	testMultipartObjectCreation(c, create)
-	testMultipartObjectAbort(c, create)
+// Return pointer to testOneByteReadEOF{}
+func newTestReaderEOF(data []byte) io.Reader {
+	return &testOneByteReadEOF{false, data}
+}
+
+// OneByteReadEOF - implements io.Reader which returns 1 byte along with io.EOF error.
+type testOneByteReadEOF struct {
+	eof  bool
+	data []byte
+}
+
+func (r *testOneByteReadEOF) Read(p []byte) (n int, err error) {
+	if r.eof {
+		return 0, io.EOF
+	}
+	n = copy(p, r.data)
+	r.eof = true
+	return n, io.EOF
+}
+
+// Return pointer to testOneByteReadNoEOF{}
+func newTestReaderNoEOF(data []byte) io.Reader {
+	return &testOneByteReadNoEOF{false, data}
+}
+
+// testOneByteReadNoEOF - implements io.Reader which returns 1 byte and nil error, but
+// returns io.EOF on the next Read().
+type testOneByteReadNoEOF struct {
+	eof  bool
+	data []byte
+}
+
+func (r *testOneByteReadNoEOF) Read(p []byte) (n int, err error) {
+	if r.eof {
+		return 0, io.EOF
+	}
+	n = copy(p, r.data)
+	r.eof = true
+	return n, nil
+}
+
+type ObjectLayerAPISuite struct{}
+
+var _ = Suite(&ObjectLayerAPISuite{})
+
+// Wrapper for calling testMakeBucket for both XL and FS.
+func (s *ObjectLayerAPISuite) TestMakeBucket(c *C) {
+	ExecObjectLayerTest(c, testMakeBucket)
 }
 
 // Tests validate bucket creation.
-func testMakeBucket(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testMakeBucket(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	err := obj.MakeBucket("bucket-unknown")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+}
 
+// Wrapper for calling testMultipartObjectCreation for both XL and FS.
+func (s *ObjectLayerAPISuite) TestMultipartObjectCreation(c *C) {
+	ExecObjectLayerTest(c, testMultipartObjectCreation)
 }
 
 // Tests validate creation of part files during Multipart operation.
-func testMultipartObjectCreation(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testMultipartObjectCreation(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	err := obj.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 	uploadID, err := obj.NewMultipartUpload("bucket", "key", nil)
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 	// Create a byte array of 5MB.
 	data := bytes.Repeat([]byte("0123456789abcdef"), 5*1024*1024/16)
 	completedParts := completeMultipartUpload{}
@@ -70,22 +110,38 @@ func testMultipartObjectCreation(c *check.C, create func() ObjectLayer) {
 
 		var calculatedMD5sum string
 		calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(data)), bytes.NewBuffer(data), expectedMD5Sumhex)
-		c.Assert(err, check.IsNil)
-		c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
+		if err != nil {
+			c.Errorf("%s: <ERROR> %s", instanceType, err)
+		}
+		if calculatedMD5sum != expectedMD5Sumhex {
+			c.Errorf("MD5 Mismatch")
+		}
 		completedParts.Parts = append(completedParts.Parts, completePart{PartNumber: i, ETag: calculatedMD5sum})
 	}
 	md5Sum, err := obj.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts)
-	c.Assert(err, check.IsNil)
-	c.Assert(md5Sum, check.Equals, "7d364cb728ce42a74a96d22949beefb2-10")
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if md5Sum != "7d364cb728ce42a74a96d22949beefb2-10" {
+		c.Errorf("Md5 mismtch")
+	}
+}
+
+// Wrapper for calling testMultipartObjectAbort for both XL and FS.
+func (s *ObjectLayerAPISuite) TestMultipartObjectAbort(c *C) {
+	ExecObjectLayerTest(c, testMultipartObjectAbort)
 }
 
 // Tests validate abortion of Multipart operation.
-func testMultipartObjectAbort(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testMultipartObjectAbort(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	err := obj.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 	uploadID, err := obj.NewMultipartUpload("bucket", "key", nil)
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
 	parts := make(map[int]string)
 	metadata := make(map[string]string)
@@ -103,20 +159,32 @@ func testMultipartObjectAbort(c *check.C, create func() ObjectLayer) {
 		metadata["md5"] = expectedMD5Sumhex
 		var calculatedMD5sum string
 		calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
-		c.Assert(err, check.IsNil)
-		c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if calculatedMD5sum != expectedMD5Sumhex {
+			c.Errorf("Md5 Mismatch")
+		}
 		parts[i] = expectedMD5Sumhex
 	}
 	err = obj.AbortMultipartUpload("bucket", "key", uploadID)
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+}
+
+// Wrapper for calling testMultipleObjectCreation for both XL and FS.
+func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(c *C) {
+	ExecObjectLayerTest(c, testMultipleObjectCreation)
 }
 
 // Tests validate object creation.
-func testMultipleObjectCreation(c *check.C, create func() ObjectLayer) {
+func testMultipleObjectCreation(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	objects := make(map[string][]byte)
-	obj := create()
 	err := obj.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 	for i := 0; i < 10; i++ {
 		randomPerm := rand.Perm(100)
 		randomString := ""
@@ -134,299 +202,598 @@ func testMultipleObjectCreation(c *check.C, create func() ObjectLayer) {
 		metadata["md5Sum"] = expectedMD5Sumhex
 		var md5Sum string
 		md5Sum, err = obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata)
-		c.Assert(err, check.IsNil)
-		c.Assert(md5Sum, check.Equals, expectedMD5Sumhex)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if md5Sum != expectedMD5Sumhex {
+			c.Errorf("Md5 Mismatch")
+		}
 	}
 
 	for key, value := range objects {
 		var byteBuffer bytes.Buffer
 		err = obj.GetObject("bucket", key, 0, int64(len(value)), &byteBuffer)
-		c.Assert(err, check.IsNil)
-		c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if !bytes.Equal(byteBuffer.Bytes(), value) {
+			c.Errorf("%s: Mismatch of GetObject data with the expected one.", instanceType)
+		}
 
 		objInfo, err := obj.GetObjectInfo("bucket", key)
-		c.Assert(err, check.IsNil)
-		c.Assert(objInfo.Size, check.Equals, int64(len(value)))
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if objInfo.Size != int64(len(value)) {
+			c.Errorf("%s: Size mismatch of the GetObject data.", instanceType)
+		}
+
 	}
 }
 
+// Wrapper for calling TestPaging for both XL and FS.
+func (s *ObjectLayerAPISuite) TestPaging(c *C) {
+	ExecObjectLayerTest(c, testPaging)
+}
+
 // Tests validate creation of objects and the order of listing using various filters for ListObjects operation.
-func testPaging(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	obj.MakeBucket("bucket")
 	result, err := obj.ListObjects("bucket", "", "", "", 0)
-	c.Assert(err, check.IsNil)
-	c.Assert(len(result.Objects), check.Equals, 0)
-	c.Assert(result.IsTruncated, check.Equals, false)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if len(result.Objects) != 0 {
+		c.Errorf("%s: Number of objects in the result different from expected value.", instanceType)
+	}
+	if result.IsTruncated != false {
+		c.Errorf("%s: Expected IsTruncated to be `false`, but instead found it to be `%v`", instanceType, result.IsTruncated)
+	}
+
+	uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
 	// check before paging occurs.
 	for i := 0; i < 5; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject("bucket", key, int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
-		c.Assert(err, check.IsNil)
+		_, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
 
 		result, err = obj.ListObjects("bucket", "", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, i+1)
-		c.Assert(result.IsTruncated, check.Equals, false)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if len(result.Objects) != i+1 {
+			c.Errorf("%s: Expected length of objects to be %d, instead found to be %d", instanceType, len(result.Objects), i+1)
+		}
+		if result.IsTruncated != false {
+			c.Errorf("%s: Expected IsTruncated to be `false`, but instead found it to be `%v`", instanceType, result.IsTruncated)
+		}
 	}
+
 	// check after paging occurs pages work.
 	for i := 6; i <= 10; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject("bucket", key, int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
-		c.Assert(err, check.IsNil)
+		_, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
 		result, err = obj.ListObjects("bucket", "obj", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, 5)
-		c.Assert(result.IsTruncated, check.Equals, true)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if len(result.Objects) != 5 {
+			c.Errorf("%s: Expected length of objects to be %d, instead found to be %d", instanceType, 5, len(result.Objects))
+		}
+		if result.IsTruncated != true {
+			c.Errorf("%s: Expected IsTruncated to be `true`, but instead found it to be `%v`", instanceType, result.IsTruncated)
+		}
 	}
 	// check paging with prefix at end returns less objects.
 	{
-		_, err = obj.PutObject("bucket", "newPrefix", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
-		c.Assert(err, check.IsNil)
-		_, err = obj.PutObject("bucket", "newPrefix2", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
-		c.Assert(err, check.IsNil)
+		_, err = obj.PutObject("bucket", "newPrefix", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		_, err = obj.PutObject("bucket", "newPrefix2", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
 		result, err = obj.ListObjects("bucket", "new", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, 2)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if len(result.Objects) != 2 {
+			c.Errorf("%s: Expected length of objects to be %d, instead found to be %d", instanceType, 2, len(result.Objects))
+		}
 	}
 
 	// check ordering of pages.
 	{
 		result, err = obj.ListObjects("bucket", "", "", "", 1000)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if result.Objects[0].Name != "newPrefix" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[0].Name)
+		}
+		if result.Objects[1].Name != "newPrefix2" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[1].Name)
+		}
+		if result.Objects[2].Name != "obj0" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[2].Name)
+		}
+		if result.Objects[3].Name != "obj1" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[3].Name)
+		}
+		if result.Objects[4].Name != "obj10" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[4].Name)
+		}
 	}
 
 	// check delimited results with delimiter and prefix.
 	{
-		_, err = obj.PutObject("bucket", "this/is/delimited", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
-		c.Assert(err, check.IsNil)
-		_, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
-		c.Assert(err, check.IsNil)
+		_, err = obj.PutObject("bucket", "this/is/delimited", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		_, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
 		result, err = obj.ListObjects("bucket", "this/is/", "", "/", 10)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, 1)
-		c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if len(result.Objects) != 1 {
+			c.Errorf("%s: Expected the number of objects in the result to be %d, but instead found %d", instanceType, 1, len(result.Objects))
+		}
+		if result.Prefixes[0] != "this/is/also/" {
+			c.Errorf("%s: Expected prefix to be `%s`, but instead found `%s`", instanceType, "this/is/also/", result.Prefixes[0])
+		}
 	}
 
 	// check delimited results with delimiter without prefix.
 	{
 		result, err = obj.ListObjects("bucket", "", "", "/", 1000)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
-		c.Assert(result.Prefixes[0], check.Equals, "this/")
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+
+		if result.Objects[0].Name != "newPrefix" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[0].Name)
+		}
+		if result.Objects[1].Name != "newPrefix2" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[1].Name)
+		}
+		if result.Objects[2].Name != "obj0" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[2].Name)
+		}
+		if result.Objects[3].Name != "obj1" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[3].Name)
+		}
+		if result.Objects[4].Name != "obj10" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[4].Name)
+		}
+		if result.Prefixes[0] != "this/" {
+			c.Errorf("%s: Expected the prefix to be `%s`, but instead found `%s`", instanceType, "this/", result.Prefixes[0])
+		}
 	}
 
 	// check results with Marker.
 	{
 
 		result, err = obj.ListObjects("bucket", "", "newPrefix", "", 3)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[1].Name, check.Equals, "obj0")
-		c.Assert(result.Objects[2].Name, check.Equals, "obj1")
-
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if result.Objects[0].Name != "newPrefix2" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix2", result.Objects[0].Name)
+		}
+		if result.Objects[1].Name != "obj0" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj0", result.Objects[1].Name)
+		}
+		if result.Objects[2].Name != "obj1" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj1", result.Objects[2].Name)
+		}
 	}
 	// check ordering of results with prefix.
 	{
 		result, err = obj.ListObjects("bucket", "obj", "", "", 1000)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Name, check.Equals, "obj0")
-		c.Assert(result.Objects[1].Name, check.Equals, "obj1")
-		c.Assert(result.Objects[2].Name, check.Equals, "obj10")
-		c.Assert(result.Objects[3].Name, check.Equals, "obj2")
-		c.Assert(result.Objects[4].Name, check.Equals, "obj3")
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if result.Objects[0].Name != "obj0" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj0", result.Objects[0].Name)
+		}
+		if result.Objects[1].Name != "obj1" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj1", result.Objects[1].Name)
+		}
+		if result.Objects[2].Name != "obj10" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj10", result.Objects[2].Name)
+		}
+		if result.Objects[3].Name != "obj2" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj2", result.Objects[3].Name)
+		}
+		if result.Objects[4].Name != "obj3" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj3", result.Objects[4].Name)
+		}
 	}
 	// check ordering of results with prefix and no paging.
 	{
 		result, err = obj.ListObjects("bucket", "new", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+		if err != nil {
+			c.Fatalf("%s: <ERROR> %s", instanceType, err)
+		}
+		if result.Objects[0].Name != "newPrefix" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[0].Name)
+		}
+		if result.Objects[1].Name != "newPrefix2" {
+			c.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix2", result.Objects[0].Name)
+		}
 	}
 }
 
+// Wrapper for calling testObjectOverwriteWorks for both XL and FS.
+func (s *ObjectLayerAPISuite) TestObjectOverwriteWorks(c *C) {
+	ExecObjectLayerTest(c, testObjectOverwriteWorks)
+}
+
 // Tests validate overwriting of an existing object.
-func testObjectOverwriteWorks(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	err := obj.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
 	_, err = obj.PutObject("bucket", "object", int64(len("The list of parts was not in ascending order. The parts list must be specified in order by part number.")), bytes.NewBufferString("The list of parts was not in ascending order. The parts list must be specified in order by part number."), nil)
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
-	length := int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."))
-	_, err = obj.PutObject("bucket", "object", length, bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
-	c.Assert(err, check.IsNil)
+	uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
+	length := int64(len(uploadContent))
+	_, err = obj.PutObject("bucket", "object", length, bytes.NewBufferString(uploadContent), nil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
 	var bytesBuffer bytes.Buffer
 	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer)
-	c.Assert(err, check.IsNil)
-	c.Assert(string(bytesBuffer.Bytes()), check.Equals, "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if string(bytesBuffer.Bytes()) != "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." {
+		c.Errorf("%s: Invalid upload ID error mismatch.", instanceType)
+	}
+}
+
+// Wrapper for calling testNonExistantBucketOperations for both XL and FS.
+func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(c *C) {
+	ExecObjectLayerTest(c, testNonExistantBucketOperations)
 }
 
 // Tests validate that bucket operation on non-existent bucket fails.
-func testNonExistantBucketOperations(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	_, err := obj.PutObject("bucket1", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(err.Error(), check.Equals, "Bucket not found: bucket1")
+	if err == nil {
+		c.Fatal("Expected error but found nil")
+	}
+	if err.Error() != "Bucket not found: bucket1" {
+		c.Errorf("%s: Expected the error msg to be `%s`, but instead found `%s`", instanceType, "Bucket not found: bucket1", err.Error())
+	}
+}
+
+// Wrapper for calling testBucketRecreateFails for both XL and FS.
+func (s *ObjectLayerAPISuite) TestBucketRecreateFails(c *C) {
+	ExecObjectLayerTest(c, testBucketRecreateFails)
 }
 
 // Tests validate that recreation of the bucket fails.
-func testBucketRecreateFails(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testBucketRecreateFails(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	err := obj.MakeBucket("string")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 	err = obj.MakeBucket("string")
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(err.Error(), check.Equals, "Bucket exists: string")
+	if err == nil {
+		c.Fatalf("%s: Expected error but found nil.", instanceType)
+	}
+
+	if err.Error() != "Bucket exists: string" {
+		c.Errorf("%s: Expected the error message to be `%s`, but instead found `%s`", instanceType, "Bucket exists: string", err.Error())
+	}
+}
+
+// Wrapper for calling testPutObject for both XL and FS.
+func (s *ObjectLayerAPISuite) TestPutObject(c *C) {
+	ExecObjectLayerTest(c, testPutObject)
+}
+
+// Tests validate PutObject without prefix.
+func testPutObject(obj ObjectLayer, instanceType string, c TestErrHandler) {
+	content := []byte("testcontent")
+	length := int64(len(content))
+	readerEOF := newTestReaderEOF(content)
+	readerNoEOF := newTestReaderNoEOF(content)
+	err := obj.MakeBucket("bucket")
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+
+	var bytesBuffer1 bytes.Buffer
+	_, err = obj.PutObject("bucket", "object", length, readerEOF, nil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer1)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if len(bytesBuffer1.Bytes()) != len(content) {
+		c.Errorf("%s: Expected content length to be `%d`, but instead found `%d`", instanceType, len(content), len(bytesBuffer1.Bytes()))
+	}
+
+	var bytesBuffer2 bytes.Buffer
+	_, err = obj.PutObject("bucket", "object", length, readerNoEOF, nil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	err = obj.GetObject("bucket", "object", 0, length, &bytesBuffer2)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if len(bytesBuffer2.Bytes()) != len(content) {
+		c.Errorf("%s: Expected content length to be `%d`, but instead found `%d`", instanceType, len(content), len(bytesBuffer2.Bytes()))
+	}
+}
+
+// Wrapper for calling testPutObjectInSubdir for both XL and FS.
+func (s *ObjectLayerAPISuite) TestPutObjectInSubdir(c *C) {
+	ExecObjectLayerTest(c, testPutObjectInSubdir)
 }
 
 // Tests validate PutObject with subdirectory prefix.
-func testPutObjectInSubdir(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testPutObjectInSubdir(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	err := obj.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
-	length := int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."))
-	_, err = obj.PutObject("bucket", "dir1/dir2/object", length, bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
-	c.Assert(err, check.IsNil)
+	uploadContent := `The specified multipart upload does not exist. The upload ID might be invalid, or the multipart
+ upload might have been aborted or completed.`
+	length := int64(len(uploadContent))
+	_, err = obj.PutObject("bucket", "dir1/dir2/object", length, bytes.NewBufferString(uploadContent), nil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
 	var bytesBuffer bytes.Buffer
 	err = obj.GetObject("bucket", "dir1/dir2/object", 0, length, &bytesBuffer)
-	c.Assert(err, check.IsNil)
-	c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."))
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if len(bytesBuffer.Bytes()) != len(uploadContent) {
+		c.Errorf("%s: Expected length of downloaded data to be `%d`, but instead found `%d`",
+			instanceType, len(uploadContent), len(bytesBuffer.Bytes()))
+	}
+}
+
+// Wrapper for calling testListBuckets for both XL and FS.
+func (s *ObjectLayerAPISuite) TestListBuckets(c *C) {
+	ExecObjectLayerTest(c, testListBuckets)
 }
 
 // Tests validate ListBuckets.
-func testListBuckets(c *check.C, create func() ObjectLayer) {
-	obj := create()
-
+func testListBuckets(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	// test empty list.
 	buckets, err := obj.ListBuckets()
-	c.Assert(err, check.IsNil)
-	c.Assert(len(buckets), check.Equals, 0)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if len(buckets) != 0 {
+		c.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 0, len(buckets))
+	}
 
 	// add one and test exists.
 	err = obj.MakeBucket("bucket1")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
 	buckets, err = obj.ListBuckets()
-	c.Assert(len(buckets), check.Equals, 1)
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if len(buckets) != 1 {
+		c.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 1, len(buckets))
+	}
 
 	// add two and test exists.
 	err = obj.MakeBucket("bucket2")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
 	buckets, err = obj.ListBuckets()
-	c.Assert(len(buckets), check.Equals, 2)
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if len(buckets) != 2 {
+		c.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 2, len(buckets))
+	}
 
 	// add three and test exists + prefix.
 	err = obj.MakeBucket("bucket22")
 
 	buckets, err = obj.ListBuckets()
-	c.Assert(len(buckets), check.Equals, 3)
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if len(buckets) != 3 {
+		c.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 3, len(buckets))
+	}
+}
+
+// Wrapper for calling testListBucketsOrder for both XL and FS.
+func (s *ObjectLayerAPISuite) TestListBucketsOrder(c *C) {
+	ExecObjectLayerTest(c, testListBucketsOrder)
 }
 
 // Tests validate the order of result of ListBuckets.
-func testListBucketsOrder(c *check.C, create func() ObjectLayer) {
+func testListBucketsOrder(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	// if implementation contains a map, order of map keys will vary.
 	// this ensures they return in the same order each time.
-	for i := 0; i < 10; i++ {
-		obj := create()
-		// add one and test exists.
-		err := obj.MakeBucket("bucket1")
-		c.Assert(err, check.IsNil)
-		err = obj.MakeBucket("bucket2")
-		c.Assert(err, check.IsNil)
-		buckets, err := obj.ListBuckets()
-		c.Assert(err, check.IsNil)
-		c.Assert(len(buckets), check.Equals, 2)
-		c.Assert(buckets[0].Name, check.Equals, "bucket1")
-		c.Assert(buckets[1].Name, check.Equals, "bucket2")
+	// add one and test exists.
+	err := obj.MakeBucket("bucket1")
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
+	err = obj.MakeBucket("bucket2")
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	buckets, err := obj.ListBuckets()
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	if len(buckets) != 2 {
+		c.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 2, len(buckets))
+	}
+
+	if buckets[0].Name != "bucket1" {
+		c.Errorf("%s: Expected bucket name to be `%s`, but instead found `%s`", instanceType, "bucket1", buckets[0].Name)
+	}
+	if buckets[1].Name != "bucket2" {
+		c.Errorf("%s: Expected bucket name to be `%s`, but instead found `%s`", instanceType, "bucket2", buckets[1].Name)
+	}
+}
+
+// Wrapper for calling testListObjectsTestsForNonExistantBucket for both XL and FS.
+func (s *ObjectLayerAPISuite) TestListObjectsTestsForNonExistantBucket(c *C) {
+	ExecObjectLayerTest(c, testListObjectsTestsForNonExistantBucket)
 }
 
 // Tests validate that ListObjects operation on a non-existent bucket fails as expected.
-func testListObjectsTestsForNonExistantBucket(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testListObjectsTestsForNonExistantBucket(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	result, err := obj.ListObjects("bucket", "", "", "", 1000)
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(result.IsTruncated, check.Equals, false)
-	c.Assert(len(result.Objects), check.Equals, 0)
-	c.Assert(err.Error(), check.Equals, "Bucket not found: bucket")
+	if err == nil {
+		c.Fatalf("%s: Expected error but found nil.", instanceType)
+	}
+	if len(result.Objects) != 0 {
+		c.Fatalf("%s: Expected number of objects in the result to be `%d`, but instead found `%d`", instanceType, 0, len(result.Objects))
+	}
+	if result.IsTruncated != false {
+		c.Fatalf("%s: Expected IsTruncated to be `false`, but instead found it to be `%v`", instanceType, result.IsTruncated)
+	}
+	if err.Error() != "Bucket not found: bucket" {
+		c.Errorf("%s: Expected the error msg to be `%s`, but instead found `%s`", instanceType, "Bucket not found: bucket", err.Error())
+	}
+}
+
+// Wrapper for calling testNonExistantObjectInBucket for both XL and FS.
+func (s *ObjectLayerAPISuite) TestNonExistantObjectInBucket(c *C) {
+	ExecObjectLayerTest(c, testNonExistantObjectInBucket)
 }
 
 // Tests validate that GetObject fails on a non-existent bucket as expected.
-func testNonExistantObjectInBucket(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	err := obj.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
-	var bytesBuffer bytes.Buffer
-	err = obj.GetObject("bucket", "dir1", 0, 10, &bytesBuffer)
-	c.Assert(err, check.Not(check.IsNil))
+	_, err = obj.GetObjectInfo("bucket", "dir1")
+	if err == nil {
+		c.Fatalf("%s: Expected error but found nil", instanceType)
+	}
 	switch err := err.(type) {
 	case ObjectNotFound:
-		c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
+		if err.Error() != "Object not found: bucket#dir1" {
+			c.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket#dir1", err.Error())
+		}
 	default:
-		c.Assert(err, check.Equals, "fails")
+		if err.Error() != "fails" {
+			c.Errorf("%s: Expected the Error message to be `%s`, but instead found it to be `%s`", instanceType, "fails", err.Error())
+		}
 	}
 }
 
+// Wrapper for calling testGetDirectoryReturnsObjectNotFound for both XL and FS.
+func (s *ObjectLayerAPISuite) TestGetDirectoryReturnsObjectNotFound(c *C) {
+	ExecObjectLayerTest(c, testGetDirectoryReturnsObjectNotFound)
+}
+
 // Tests validate that GetObject on an existing directory fails as expected.
-func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() ObjectLayer) {
-	obj := create()
+func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, c TestErrHandler) {
 	err := obj.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
 
-	_, err = obj.PutObject("bucket", "dir1/dir3/object", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."), nil)
-	c.Assert(err, check.IsNil)
+	_, err = obj.PutObject("bucket", "dir1/dir3/object",
+		int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")),
+		bytes.NewBufferString("One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."), nil)
 
-	var bytesBuffer bytes.Buffer
-	err = obj.GetObject("bucket", "dir1", 0, 10, &bytesBuffer)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+
+	_, err = obj.GetObjectInfo("bucket", "dir1")
 	switch err := err.(type) {
 	case ObjectNotFound:
-		c.Assert(err.Bucket, check.Equals, "bucket")
-		c.Assert(err.Object, check.Equals, "dir1")
+		if err.Bucket != "bucket" {
+			c.Errorf("%s: Expected the bucket name in the error message to be `%s`, but instead found `%s`", instanceType, "bucket", err.Bucket)
+		}
+		if err.Object != "dir1" {
+			c.Errorf("%s: Expected the object name in the error message to be `%s`, but instead found `%s`", instanceType, "dir1", err.Object)
+		}
 	default:
-		// force a failure with a line number.
-		c.Assert(err, check.Equals, "ObjectNotFound")
+		if err.Error() != "ObjectNotFound" {
+			c.Errorf("%s: Expected the error message to be `%s`, but instead found `%s`", instanceType, "ObjectNotFound", err.Error())
+		}
 	}
 
-	err = obj.GetObject("bucket", "dir1/", 0, 10, &bytesBuffer)
+	_, err = obj.GetObjectInfo("bucket", "dir1/")
 	switch err := err.(type) {
 	case ObjectNameInvalid:
-		c.Assert(err.Bucket, check.Equals, "bucket")
-		c.Assert(err.Object, check.Equals, "dir1/")
+		if err.Bucket != "bucket" {
+			c.Errorf("%s: Expected the bucket name in the error message to be `%s`, but instead found `%s`", instanceType, "bucket", err.Bucket)
+		}
+		if err.Object != "dir1/" {
+			c.Errorf("%s: Expected the object name in the error message to be `%s`, but instead found `%s`", instanceType, "dir1/", err.Object)
+		}
 	default:
 		// force a failure with a line number.
-		c.Assert(err, check.Equals, "ObjectNotFound")
+		if err.Error() != "ObjectNotFound" {
+			c.Errorf("%s: Expected the error message to be `%s`, but instead found `%s`", instanceType, "ObjectNotFound", err.Error())
+		}
 	}
 }
 
-// Tests valdiate the default ContentType.
-func testDefaultContentType(c *check.C, create func() ObjectLayer) {
-	obj := create()
-	err := obj.MakeBucket("bucket")
-	c.Assert(err, check.IsNil)
-
-	// Test empty.
-	_, err = obj.PutObject("bucket", "one", int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")), bytes.NewBufferString("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."), nil)
-	c.Assert(err, check.IsNil)
-	objInfo, err := obj.GetObjectInfo("bucket", "one")
-	c.Assert(err, check.IsNil)
-	c.Assert(objInfo.ContentType, check.Equals, "application/octet-stream")
+// Wrapper for calling testContentType for both XL and FS.
+func (s *ObjectLayerAPISuite) TestContentType(c *C) {
+	ExecObjectLayerTest(c, testContentType)
+}
+
+// Test content-type.
+func testContentType(obj ObjectLayer, instanceType string, c TestErrHandler) {
+	err := obj.MakeBucket("bucket")
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
+	// Test empty.
+	_, err = obj.PutObject("bucket", "minio.png", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil)
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+	objInfo, err := obj.GetObjectInfo("bucket", "minio.png")
+	if err != nil {
+		c.Fatalf("%s: <ERROR> %s", instanceType, err)
+	}
+
+	if objInfo.ContentType != "image/png" {
+		c.Errorf("%s: Expected Content type to be `%s`, but instead found `%s`", instanceType, "image/png", objInfo.ContentType)
+	}
 }

pkg/crypto/sha512/sha512-avx-asm_linux_amd64.S

@@ -1,686 +0,0 @@
-########################################################################
-# Implement fast SHA-512 with AVX instructions. (x86_64)
-#
-# Copyright (C) 2013 Intel Corporation.
-#
-# Authors:
-#     James Guilford <james.guilford@intel.com>
-#     Kirk Yap <kirk.s.yap@intel.com>
-#     David Cote <david.m.cote@intel.com>
-#     Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses.  You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-#     Redistribution and use in source and binary forms, with or
-#     without modification, are permitted provided that the following
-#     conditions are met:
-#
-#      - Redistributions of source code must retain the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer.
-#
-#      - Redistributions in binary form must reproduce the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer in the documentation and/or other materials
-#        provided with the distribution.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-########################################################################
-#
-# This code is described in an Intel White-Paper:
-# "Fast SHA-512 Implementations on Intel Architecture Processors"
-#
-# To find it, surf to http://www.intel.com/p/en_US/embedded
-# and search for that title.
-#
-########################################################################
-# Using this part of Minio codebase under the license
-# Apache License Version 2.0 with modifications
-##
-
-#ifdef HAS_AVX
-#ifndef ENTRY
-#define ENTRY(name) \
-        .globl name             ; \
-        .align 4,0x90           ; \
-        name:
-#endif
-
-#ifndef END
-#define END(name) \
-        .size name, .-name
-#endif
-
-#ifndef ENDPROC
-#define ENDPROC(name) \
-        .type name, @function   ; \
-        END(name)
-#endif
-
-#define NUM_INVALID		100
-
-#define TYPE_R32		0
-#define TYPE_R64		1
-#define TYPE_XMM		2
-#define TYPE_INVALID	100
-
-	.macro R32_NUM opd r32
-	\opd = NUM_INVALID
-	.ifc \r32,%eax
-	\opd = 0
-	.endif
-	.ifc \r32,%ecx
-	\opd = 1
-	.endif
-	.ifc \r32,%edx
-	\opd = 2
-	.endif
-	.ifc \r32,%ebx
-	\opd = 3
-	.endif
-	.ifc \r32,%esp
-	\opd = 4
-	.endif
-	.ifc \r32,%ebp
-	\opd = 5
-	.endif
-	.ifc \r32,%esi
-	\opd = 6
-	.endif
-	.ifc \r32,%edi
-	\opd = 7
-	.endif
-#ifdef X86_64
-	.ifc \r32,%r8d
-	\opd = 8
-	.endif
-	.ifc \r32,%r9d
-	\opd = 9
-	.endif
-	.ifc \r32,%r10d
-	\opd = 10
-	.endif
-	.ifc \r32,%r11d
-	\opd = 11
-	.endif
-	.ifc \r32,%r12d
-	\opd = 12
-	.endif
-	.ifc \r32,%r13d
-	\opd = 13
-	.endif
-	.ifc \r32,%r14d
-	\opd = 14
-	.endif
-	.ifc \r32,%r15d
-	\opd = 15
-	.endif
-#endif
-	.endm
-
-	.macro R64_NUM opd r64
-	\opd = NUM_INVALID
-#ifdef X86_64
-	.ifc \r64,%rax
-	\opd = 0
-	.endif
-	.ifc \r64,%rcx
-	\opd = 1
-	.endif
-	.ifc \r64,%rdx
-	\opd = 2
-	.endif
-	.ifc \r64,%rbx
-	\opd = 3
-	.endif
-	.ifc \r64,%rsp
-	\opd = 4
-	.endif
-	.ifc \r64,%rbp
-	\opd = 5
-	.endif
-	.ifc \r64,%rsi
-	\opd = 6
-	.endif
-	.ifc \r64,%rdi
-	\opd = 7
-	.endif
-	.ifc \r64,%r8
-	\opd = 8
-	.endif
-	.ifc \r64,%r9
-	\opd = 9
-	.endif
-	.ifc \r64,%r10
-	\opd = 10
-	.endif
-	.ifc \r64,%r11
-	\opd = 11
-	.endif
-	.ifc \r64,%r12
-	\opd = 12
-	.endif
-	.ifc \r64,%r13
-	\opd = 13
-	.endif
-	.ifc \r64,%r14
-	\opd = 14
-	.endif
-	.ifc \r64,%r15
-	\opd = 15
-	.endif
-#endif
-	.endm
-
-	.macro XMM_NUM opd xmm
-	\opd = NUM_INVALID
-	.ifc \xmm,%xmm0
-	\opd = 0
-	.endif
-	.ifc \xmm,%xmm1
-	\opd = 1
-	.endif
-	.ifc \xmm,%xmm2
-	\opd = 2
-	.endif
-	.ifc \xmm,%xmm3
-	\opd = 3
-	.endif
-	.ifc \xmm,%xmm4
-	\opd = 4
-	.endif
-	.ifc \xmm,%xmm5
-	\opd = 5
-	.endif
-	.ifc \xmm,%xmm6
-	\opd = 6
-	.endif
-	.ifc \xmm,%xmm7
-	\opd = 7
-	.endif
-	.ifc \xmm,%xmm8
-	\opd = 8
-	.endif
-	.ifc \xmm,%xmm9
-	\opd = 9
-	.endif
-	.ifc \xmm,%xmm10
-	\opd = 10
-	.endif
-	.ifc \xmm,%xmm11
-	\opd = 11
-	.endif
-	.ifc \xmm,%xmm12
-	\opd = 12
-	.endif
-	.ifc \xmm,%xmm13
-	\opd = 13
-	.endif
-	.ifc \xmm,%xmm14
-	\opd = 14
-	.endif
-	.ifc \xmm,%xmm15
-	\opd = 15
-	.endif
-	.endm
-
-	.macro TYPE type reg
-	R32_NUM reg_type_r32 \reg
-	R64_NUM reg_type_r64 \reg
-	XMM_NUM reg_type_xmm \reg
-	.if reg_type_r64 <> NUM_INVALID
-	\type = TYPE_R64
-	.elseif reg_type_r32 <> NUM_INVALID
-	\type = TYPE_R32
-	.elseif reg_type_xmm <> NUM_INVALID
-	\type = TYPE_XMM
-	.else
-	\type = TYPE_INVALID
-	.endif
-	.endm
-
-	.macro PFX_OPD_SIZE
-	.byte 0x66
-	.endm
-
-	.macro PFX_REX opd1 opd2 W=0
-	.if ((\opd1 | \opd2) & 8) || \W
-	.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
-	.endif
-	.endm
-
-	.macro MODRM mod opd1 opd2
-	.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
-	.endm
-
-	.macro PSHUFB_XMM xmm1 xmm2
-	XMM_NUM pshufb_opd1 \xmm1
-	XMM_NUM pshufb_opd2 \xmm2
-	PFX_OPD_SIZE
-	PFX_REX pshufb_opd1 pshufb_opd2
-	.byte 0x0f, 0x38, 0x00
-	MODRM 0xc0 pshufb_opd1 pshufb_opd2
-	.endm
-
-	.macro PCLMULQDQ imm8 xmm1 xmm2
-	XMM_NUM clmul_opd1 \xmm1
-	XMM_NUM clmul_opd2 \xmm2
-	PFX_OPD_SIZE
-	PFX_REX clmul_opd1 clmul_opd2
-	.byte 0x0f, 0x3a, 0x44
-	MODRM 0xc0 clmul_opd1 clmul_opd2
-	.byte \imm8
-	.endm
-
-	.macro PEXTRD imm8 xmm gpr
-	R32_NUM extrd_opd1 \gpr
-	XMM_NUM extrd_opd2 \xmm
-	PFX_OPD_SIZE
-	PFX_REX extrd_opd1 extrd_opd2
-	.byte 0x0f, 0x3a, 0x16
-	MODRM 0xc0 extrd_opd1 extrd_opd2
-	.byte \imm8
-	.endm
-
-	.macro MOVQ_R64_XMM opd1 opd2
-	TYPE movq_r64_xmm_opd1_type \opd1
-	.if movq_r64_xmm_opd1_type == TYPE_XMM
-	XMM_NUM movq_r64_xmm_opd1 \opd1
-	R64_NUM movq_r64_xmm_opd2 \opd2
-	.else
-	R64_NUM movq_r64_xmm_opd1 \opd1
-	XMM_NUM movq_r64_xmm_opd2 \opd2
-	.endif
-	PFX_OPD_SIZE
-	PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
-	.if movq_r64_xmm_opd1_type == TYPE_XMM
-	.byte 0x0f, 0x7e
-	.else
-	.byte 0x0f, 0x6e
-	.endif
-	MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
-	.endm
-
-.text
-
-# Virtual Registers
-# ARG1
-msg	= %rdi
-# ARG2
-digest	= %rsi
-# ARG3
-msglen	= %rdx
-T1	= %rcx
-T2	= %r8
-a_64	= %r9
-b_64	= %r10
-c_64	= %r11
-d_64	= %r12
-e_64	= %r13
-f_64	= %r14
-g_64	= %r15
-h_64	= %rbx
-tmp0	= %rax
-
-# Local variables (stack frame)
-
-# Message Schedule
-W_SIZE = 80*8
-# W[t] + K[t] | W[t+1] + K[t+1]
-WK_SIZE = 2*8
-RSPSAVE_SIZE = 1*8
-GPRSAVE_SIZE = 5*8
-
-frame_W = 0
-frame_WK = frame_W + W_SIZE
-frame_RSPSAVE = frame_WK + WK_SIZE
-frame_GPRSAVE = frame_RSPSAVE + RSPSAVE_SIZE
-frame_size = frame_GPRSAVE + GPRSAVE_SIZE
-
-# Useful QWORD "arrays" for simpler memory references
-# MSG, DIGEST, K_t, W_t are arrays
-# WK_2(t) points to 1 of 2 qwords at frame.WK depdending on t being odd/even
-
-# Input message (arg1)
-#define MSG(i)    8*i(msg)
-
-# Output Digest (arg2)
-#define DIGEST(i) 8*i(digest)
-
-# SHA Constants (static mem)
-#define K_t(i)    8*i+K512(%rip)
-
-# Message Schedule (stack frame)
-#define W_t(i)    8*i+frame_W(%rsp)
-
-# W[t]+K[t] (stack frame)
-#define WK_2(i)   8*((i%2))+frame_WK(%rsp)
-
-.macro RotateState
-	# Rotate symbols a..h right
-	TMP   = h_64
-	h_64  = g_64
-	g_64  = f_64
-	f_64  = e_64
-	e_64  = d_64
-	d_64  = c_64
-	c_64  = b_64
-	b_64  = a_64
-	a_64  = TMP
-.endm
-
-.macro RORQ p1 p2
-	# shld is faster than ror on Sandybridge
-	shld	$(64-\p2), \p1, \p1
-.endm
-
-.macro SHA512_Round rnd
-	# Compute Round %%t
-	mov     f_64, T1          # T1 = f
-	mov     e_64, tmp0        # tmp = e
-	xor     g_64, T1          # T1 = f ^ g
-	RORQ    tmp0, 23   # 41    # tmp = e ror 23
-	and     e_64, T1          # T1 = (f ^ g) & e
-	xor     e_64, tmp0        # tmp = (e ror 23) ^ e
-	xor     g_64, T1          # T1 = ((f ^ g) & e) ^ g = CH(e,f,g)
-	idx = \rnd
-	add     WK_2(idx), T1     # W[t] + K[t] from message scheduler
-	RORQ    tmp0, 4   # 18    # tmp = ((e ror 23) ^ e) ror 4
-	xor     e_64, tmp0        # tmp = (((e ror 23) ^ e) ror 4) ^ e
-	mov     a_64, T2          # T2 = a
-	add     h_64, T1          # T1 = CH(e,f,g) + W[t] + K[t] + h
-	RORQ    tmp0, 14  # 14    # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e)
-	add     tmp0, T1          # T1 = CH(e,f,g) + W[t] + K[t] + S1(e)
-	mov     a_64, tmp0        # tmp = a
-	xor     c_64, T2          # T2 = a ^ c
-	and     c_64, tmp0        # tmp = a & c
-	and     b_64, T2          # T2 = (a ^ c) & b
-	xor     tmp0, T2          # T2 = ((a ^ c) & b) ^ (a & c) = Maj(a,b,c)
-	mov     a_64, tmp0        # tmp = a
-	RORQ    tmp0, 5  # 39     # tmp = a ror 5
-	xor     a_64, tmp0        # tmp = (a ror 5) ^ a
-	add     T1, d_64          # e(next_state) = d + T1
-	RORQ    tmp0, 6  # 34     # tmp = ((a ror 5) ^ a) ror 6
-	xor     a_64, tmp0        # tmp = (((a ror 5) ^ a) ror 6) ^ a
-	lea     (T1, T2), h_64    # a(next_state) = T1 + Maj(a,b,c)
-	RORQ    tmp0, 28  # 28    # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a)
-	add     tmp0, h_64        # a(next_state) = T1 + Maj(a,b,c) S0(a)
-	RotateState
-.endm
-
-.macro SHA512_2Sched_2Round_avx rnd
-	# Compute rounds t-2 and t-1
-	# Compute message schedule QWORDS t and t+1
-
-	#   Two rounds are computed based on the values for K[t-2]+W[t-2] and
-	# K[t-1]+W[t-1] which were previously stored at WK_2 by the message
-	# scheduler.
-	#   The two new schedule QWORDS are stored at [W_t(t)] and [W_t(t+1)].
-	# They are then added to their respective SHA512 constants at
-	# [K_t(t)] and [K_t(t+1)] and stored at dqword [WK_2(t)]
-	#   For brievity, the comments following vectored instructions only refer to
-	# the first of a pair of QWORDS.
-	# Eg. XMM4=W[t-2] really means XMM4={W[t-2]|W[t-1]}
-	#   The computation of the message schedule and the rounds are tightly
-	# stitched to take advantage of instruction-level parallelism.
-
-	idx = \rnd - 2
-	vmovdqa	W_t(idx), %xmm4		# XMM4 = W[t-2]
-	idx = \rnd - 15
-	vmovdqu	W_t(idx), %xmm5		# XMM5 = W[t-15]
-	mov	f_64, T1
-	vpsrlq	$61, %xmm4, %xmm0	# XMM0 = W[t-2]>>61
-	mov	e_64, tmp0
-	vpsrlq	$1, %xmm5, %xmm6	# XMM6 = W[t-15]>>1
-	xor	g_64, T1
-	RORQ	tmp0, 23 # 41
-	vpsrlq	$19, %xmm4, %xmm1	# XMM1 = W[t-2]>>19
-	and	e_64, T1
-	xor	e_64, tmp0
-	vpxor	%xmm1, %xmm0, %xmm0	# XMM0 = W[t-2]>>61 ^ W[t-2]>>19
-	xor	g_64, T1
-	idx = \rnd
-	add	WK_2(idx), T1#
-	vpsrlq	$8, %xmm5, %xmm7	# XMM7 = W[t-15]>>8
-	RORQ	tmp0, 4 # 18
-	vpsrlq	$6, %xmm4, %xmm2	# XMM2 = W[t-2]>>6
-	xor	e_64, tmp0
-	mov	a_64, T2
-	add	h_64, T1
-	vpxor	%xmm7, %xmm6, %xmm6	# XMM6 = W[t-15]>>1 ^ W[t-15]>>8
-	RORQ	tmp0, 14 # 14
-	add	tmp0, T1
-	vpsrlq	$7, %xmm5, %xmm8	# XMM8 = W[t-15]>>7
-	mov	a_64, tmp0
-	xor	c_64, T2
-	vpsllq	$(64-61), %xmm4, %xmm3  # XMM3 = W[t-2]<<3
-	and	c_64, tmp0
-	and	b_64, T2
-	vpxor	%xmm3, %xmm2, %xmm2	# XMM2 = W[t-2]>>6 ^ W[t-2]<<3
-	xor	tmp0, T2
-	mov	a_64, tmp0
-	vpsllq	$(64-1), %xmm5, %xmm9	# XMM9 = W[t-15]<<63
-	RORQ	tmp0, 5 # 39
-	vpxor	%xmm9, %xmm8, %xmm8	# XMM8 = W[t-15]>>7 ^ W[t-15]<<63
-	xor	a_64, tmp0
-	add	T1, d_64
-	RORQ	tmp0, 6 # 34
-	xor	a_64, tmp0
-	vpxor	%xmm8, %xmm6, %xmm6	# XMM6 = W[t-15]>>1 ^ W[t-15]>>8 ^
-					#  W[t-15]>>7 ^ W[t-15]<<63
-	lea	(T1, T2), h_64
-	RORQ	tmp0, 28 # 28
-	vpsllq	$(64-19), %xmm4, %xmm4  # XMM4 = W[t-2]<<25
-	add	tmp0, h_64
-	RotateState
-	vpxor	%xmm4, %xmm0, %xmm0     # XMM0 = W[t-2]>>61 ^ W[t-2]>>19 ^
-					#        W[t-2]<<25
-	mov	f_64, T1
-	vpxor	%xmm2, %xmm0, %xmm0     # XMM0 = s1(W[t-2])
-	mov	e_64, tmp0
-	xor	g_64, T1
-	idx = \rnd - 16
-	vpaddq	W_t(idx), %xmm0, %xmm0  # XMM0 = s1(W[t-2]) + W[t-16]
-	idx = \rnd - 7
-	vmovdqu	W_t(idx), %xmm1		# XMM1 = W[t-7]
-	RORQ	tmp0, 23 # 41
-	and	e_64, T1
-	xor	e_64, tmp0
-	xor	g_64, T1
-	vpsllq	$(64-8), %xmm5, %xmm5   # XMM5 = W[t-15]<<56
-	idx = \rnd + 1
-	add	WK_2(idx), T1
-	vpxor	%xmm5, %xmm6, %xmm6     # XMM6 = s0(W[t-15])
-	RORQ	tmp0, 4 # 18
-	vpaddq	%xmm6, %xmm0, %xmm0     # XMM0 = s1(W[t-2]) + W[t-16] + s0(W[t-15])
-	xor	e_64, tmp0
-	vpaddq	%xmm1, %xmm0, %xmm0     # XMM0 = W[t] = s1(W[t-2]) + W[t-7] +
-					#               s0(W[t-15]) + W[t-16]
-	mov	a_64, T2
-	add	h_64, T1
-	RORQ	tmp0, 14 # 14
-	add	tmp0, T1
-	idx = \rnd
-	vmovdqa	%xmm0, W_t(idx)		# Store W[t]
-	vpaddq	K_t(idx), %xmm0, %xmm0  # Compute W[t]+K[t]
-	vmovdqa	%xmm0, WK_2(idx)	# Store W[t]+K[t] for next rounds
-	mov	a_64, tmp0
-	xor	c_64, T2
-	and	c_64, tmp0
-	and	b_64, T2
-	xor	tmp0, T2
-	mov	a_64, tmp0
-	RORQ	tmp0, 5 # 39
-	xor	a_64, tmp0
-	add	T1, d_64
-	RORQ	tmp0, 6 # 34
-	xor	a_64, tmp0
-	lea	(T1, T2), h_64
-	RORQ	tmp0, 28 # 28
-	add	tmp0, h_64
-	RotateState
-.endm
-
-########################################################################
-# void sha512_transform_avx(const void* M, void* D, u64 L)
-# Purpose: Updates the SHA512 digest stored at D with the message stored in M.
-# The size of the message pointed to by M must be an integer multiple of SHA512
-# message blocks.
-# L is the message length in SHA512 blocks
-########################################################################
-ENTRY(sha512_transform_avx)
-	cmp $0, msglen
-	je nowork
-
-	# Allocate Stack Space
-	mov	%rsp, %rax
-	sub     $frame_size, %rsp
-	and	$~(0x20 - 1), %rsp
-	mov	%rax, frame_RSPSAVE(%rsp)
-
-	# Save GPRs
-	mov     %rbx, frame_GPRSAVE(%rsp)
-	mov     %r12, frame_GPRSAVE +8*1(%rsp)
-	mov     %r13, frame_GPRSAVE +8*2(%rsp)
-	mov     %r14, frame_GPRSAVE +8*3(%rsp)
-	mov     %r15, frame_GPRSAVE +8*4(%rsp)
-
-updateblock:
-
-	# Load state variables
-	mov     DIGEST(0), a_64
-	mov     DIGEST(1), b_64
-	mov     DIGEST(2), c_64
-	mov     DIGEST(3), d_64
-	mov     DIGEST(4), e_64
-	mov     DIGEST(5), f_64
-	mov     DIGEST(6), g_64
-	mov     DIGEST(7), h_64
-
-	t = 0
-	.rept 80/2 + 1
-	# (80 rounds) / (2 rounds/iteration) + (1 iteration)
-	# +1 iteration because the scheduler leads hashing by 1 iteration
-		.if t < 2
-			# BSWAP 2 QWORDS
-			vmovdqa  XMM_QWORD_BSWAP(%rip), %xmm1
-			vmovdqu  MSG(t), %xmm0
-			vpshufb  %xmm1, %xmm0, %xmm0    # BSWAP
-			vmovdqa  %xmm0, W_t(t) # Store Scheduled Pair
-			vpaddq   K_t(t), %xmm0, %xmm0 # Compute W[t]+K[t]
-			vmovdqa  %xmm0, WK_2(t) # Store into WK for rounds
-		.elseif t < 16
-			# BSWAP 2 QWORDS# Compute 2 Rounds
-			vmovdqu  MSG(t), %xmm0
-			vpshufb  %xmm1, %xmm0, %xmm0    # BSWAP
-			SHA512_Round t-2    # Round t-2
-			vmovdqa  %xmm0, W_t(t) # Store Scheduled Pair
-			vpaddq   K_t(t), %xmm0, %xmm0 # Compute W[t]+K[t]
-			SHA512_Round t-1    # Round t-1
-			vmovdqa  %xmm0, WK_2(t)# Store W[t]+K[t] into WK
-		.elseif t < 79
-			# Schedule 2 QWORDS# Compute 2 Rounds
-			SHA512_2Sched_2Round_avx t
-		.else
-			# Compute 2 Rounds
-			SHA512_Round t-2
-			SHA512_Round t-1
-		.endif
-		t = t+2
-	.endr
-
-	# Update digest
-	add     a_64, DIGEST(0)
-	add     b_64, DIGEST(1)
-	add     c_64, DIGEST(2)
-	add     d_64, DIGEST(3)
-	add     e_64, DIGEST(4)
-	add     f_64, DIGEST(5)
-	add     g_64, DIGEST(6)
-	add     h_64, DIGEST(7)
-
-	# Advance to next message block
-	add     $16*8, msg
-	dec     msglen
-	jnz     updateblock
-
-	# Restore GPRs
-	mov     frame_GPRSAVE(%rsp),      %rbx
-	mov     frame_GPRSAVE +8*1(%rsp), %r12
-	mov     frame_GPRSAVE +8*2(%rsp), %r13
-	mov     frame_GPRSAVE +8*3(%rsp), %r14
-	mov     frame_GPRSAVE +8*4(%rsp), %r15
-
-	# Restore Stack Pointer
-	mov	frame_RSPSAVE(%rsp), %rsp
-
-nowork:
-	ret
-ENDPROC(sha512_transform_avx)
-
-########################################################################
-### Binary Data
-
-.data
-
-.align 16
-
-# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
-XMM_QWORD_BSWAP:
-	.octa 0x08090a0b0c0d0e0f0001020304050607
-
-# K[t] used in SHA512 hashing
-K512:
-	.quad 0x428a2f98d728ae22,0x7137449123ef65cd
-	.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
-	.quad 0x3956c25bf348b538,0x59f111f1b605d019
-	.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
-	.quad 0xd807aa98a3030242,0x12835b0145706fbe
-	.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
-	.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
-	.quad 0x9bdc06a725c71235,0xc19bf174cf692694
-	.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
-	.quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
-	.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
-	.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
-	.quad 0x983e5152ee66dfab,0xa831c66d2db43210
-	.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
-	.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
-	.quad 0x06ca6351e003826f,0x142929670a0e6e70
-	.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
-	.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
-	.quad 0x650a73548baf63de,0x766a0abb3c77b2a8
-	.quad 0x81c2c92e47edaee6,0x92722c851482353b
-	.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
-	.quad 0xc24b8b70d0f89791,0xc76c51a30654be30
-	.quad 0xd192e819d6ef5218,0xd69906245565a910
-	.quad 0xf40e35855771202a,0x106aa07032bbd1b8
-	.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
-	.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
-	.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
-	.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
-	.quad 0x748f82ee5defb2fc,0x78a5636f43172f60
-	.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
-	.quad 0x90befffa23631e28,0xa4506cebde82bde9
-	.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
-	.quad 0xca273eceea26619c,0xd186b8c721c0c207
-	.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
-	.quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
-	.quad 0x113f9804bef90dae,0x1b710b35131c471b
-	.quad 0x28db77f523047d84,0x32caab7b40c72493
-	.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
-	.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
-	.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
-#endif

pkg/crypto/sha512/sha512-ssse3-asm_linux_amd64.S

@@ -1,686 +0,0 @@
-########################################################################
-# Implement fast SHA-512 with SSSE3 instructions. (x86_64)
-#
-# Copyright (C) 2013 Intel Corporation.
-#
-# Authors:
-#     James Guilford <james.guilford@intel.com>
-#     Kirk Yap <kirk.s.yap@intel.com>
-#     David Cote <david.m.cote@intel.com>
-#     Tim Chen <tim.c.chen@linux.intel.com>
-#
-# This software is available to you under a choice of one of two
-# licenses.  You may choose to be licensed under the terms of the GNU
-# General Public License (GPL) Version 2, available from the file
-# COPYING in the main directory of this source tree, or the
-# OpenIB.org BSD license below:
-#
-#     Redistribution and use in source and binary forms, with or
-#     without modification, are permitted provided that the following
-#     conditions are met:
-#
-#      - Redistributions of source code must retain the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer.
-#
-#      - Redistributions in binary form must reproduce the above
-#        copyright notice, this list of conditions and the following
-#        disclaimer in the documentation and/or other materials
-#        provided with the distribution.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-########################################################################
-#
-# This code is described in an Intel White-Paper:
-# "Fast SHA-512 Implementations on Intel Architecture Processors"
-#
-# To find it, surf to http://www.intel.com/p/en_US/embedded
-# and search for that title.
-#
-########################################################################
-# Using this part of Minio codebase under the license
-# Apache License Version 2.0 with modifications
-##
-
-#ifdef HAS_SSE41
-#ifndef ENTRY
-#define ENTRY(name) \
-        .globl name             ; \
-        .align 4,0x90           ; \
-        name:
-#endif
-
-#ifndef END
-#define END(name) \
-        .size name, .-name
-#endif
-
-#ifndef ENDPROC
-#define ENDPROC(name) \
-        .type name, @function   ; \
-        END(name)
-#endif
-
-#define NUM_INVALID		100
-
-#define TYPE_R32		0
-#define TYPE_R64		1
-#define TYPE_XMM		2
-#define TYPE_INVALID	100
-
-	.macro R32_NUM opd r32
-	\opd = NUM_INVALID
-	.ifc \r32,%eax
-	\opd = 0
-	.endif
-	.ifc \r32,%ecx
-	\opd = 1
-	.endif
-	.ifc \r32,%edx
-	\opd = 2
-	.endif
-	.ifc \r32,%ebx
-	\opd = 3
-	.endif
-	.ifc \r32,%esp
-	\opd = 4
-	.endif
-	.ifc \r32,%ebp
-	\opd = 5
-	.endif
-	.ifc \r32,%esi
-	\opd = 6
-	.endif
-	.ifc \r32,%edi
-	\opd = 7
-	.endif
-#ifdef X86_64
-	.ifc \r32,%r8d
-	\opd = 8
-	.endif
-	.ifc \r32,%r9d
-	\opd = 9
-	.endif
-	.ifc \r32,%r10d
-	\opd = 10
-	.endif
-	.ifc \r32,%r11d
-	\opd = 11
-	.endif
-	.ifc \r32,%r12d
-	\opd = 12
-	.endif
-	.ifc \r32,%r13d
-	\opd = 13
-	.endif
-	.ifc \r32,%r14d
-	\opd = 14
-	.endif
-	.ifc \r32,%r15d
-	\opd = 15
-	.endif
-#endif
-	.endm
-
-	.macro R64_NUM opd r64
-	\opd = NUM_INVALID
-#ifdef X86_64
-	.ifc \r64,%rax
-	\opd = 0
-	.endif
-	.ifc \r64,%rcx
-	\opd = 1
-	.endif
-	.ifc \r64,%rdx
-	\opd = 2
-	.endif
-	.ifc \r64,%rbx
-	\opd = 3
-	.endif
-	.ifc \r64,%rsp
-	\opd = 4
-	.endif
-	.ifc \r64,%rbp
-	\opd = 5
-	.endif
-	.ifc \r64,%rsi
-	\opd = 6
-	.endif
-	.ifc \r64,%rdi
-	\opd = 7
-	.endif
-	.ifc \r64,%r8
-	\opd = 8
-	.endif
-	.ifc \r64,%r9
-	\opd = 9
-	.endif
-	.ifc \r64,%r10
-	\opd = 10
-	.endif
-	.ifc \r64,%r11
-	\opd = 11
-	.endif
-	.ifc \r64,%r12
-	\opd = 12
-	.endif
-	.ifc \r64,%r13
-	\opd = 13
-	.endif
-	.ifc \r64,%r14
-	\opd = 14
-	.endif
-	.ifc \r64,%r15
-	\opd = 15
-	.endif
-#endif
-	.endm
-
-	.macro XMM_NUM opd xmm
-	\opd = NUM_INVALID
-	.ifc \xmm,%xmm0
-	\opd = 0
-	.endif
-	.ifc \xmm,%xmm1
-	\opd = 1
-	.endif
-	.ifc \xmm,%xmm2
-	\opd = 2
-	.endif
-	.ifc \xmm,%xmm3
-	\opd = 3
-	.endif
-	.ifc \xmm,%xmm4
-	\opd = 4
-	.endif
-	.ifc \xmm,%xmm5
-	\opd = 5
-	.endif
-	.ifc \xmm,%xmm6
-	\opd = 6
-	.endif
-	.ifc \xmm,%xmm7
-	\opd = 7
-	.endif
-	.ifc \xmm,%xmm8
-	\opd = 8
-	.endif
-	.ifc \xmm,%xmm9
-	\opd = 9
-	.endif
-	.ifc \xmm,%xmm10
-	\opd = 10
-	.endif
-	.ifc \xmm,%xmm11
-	\opd = 11
-	.endif
-	.ifc \xmm,%xmm12
-	\opd = 12
-	.endif
-	.ifc \xmm,%xmm13
-	\opd = 13
-	.endif
-	.ifc \xmm,%xmm14
-	\opd = 14
-	.endif
-	.ifc \xmm,%xmm15
-	\opd = 15
-	.endif
-	.endm
-
-	.macro TYPE type reg
-	R32_NUM reg_type_r32 \reg
-	R64_NUM reg_type_r64 \reg
-	XMM_NUM reg_type_xmm \reg
-	.if reg_type_r64 <> NUM_INVALID
-	\type = TYPE_R64
-	.elseif reg_type_r32 <> NUM_INVALID
-	\type = TYPE_R32
-	.elseif reg_type_xmm <> NUM_INVALID
-	\type = TYPE_XMM
-	.else
-	\type = TYPE_INVALID
-	.endif
-	.endm
-
-	.macro PFX_OPD_SIZE
-	.byte 0x66
-	.endm
-
-	.macro PFX_REX opd1 opd2 W=0
-	.if ((\opd1 | \opd2) & 8) || \W
-	.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
-	.endif
-	.endm
-
-	.macro MODRM mod opd1 opd2
-	.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
-	.endm
-
-	.macro PSHUFB_XMM xmm1 xmm2
-	XMM_NUM pshufb_opd1 \xmm1
-	XMM_NUM pshufb_opd2 \xmm2
-	PFX_OPD_SIZE
-	PFX_REX pshufb_opd1 pshufb_opd2
-	.byte 0x0f, 0x38, 0x00
-	MODRM 0xc0 pshufb_opd1 pshufb_opd2
-	.endm
-
-	.macro PCLMULQDQ imm8 xmm1 xmm2
-	XMM_NUM clmul_opd1 \xmm1
-	XMM_NUM clmul_opd2 \xmm2
-	PFX_OPD_SIZE
-	PFX_REX clmul_opd1 clmul_opd2
-	.byte 0x0f, 0x3a, 0x44
-	MODRM 0xc0 clmul_opd1 clmul_opd2
-	.byte \imm8
-	.endm
-
-	.macro PEXTRD imm8 xmm gpr
-	R32_NUM extrd_opd1 \gpr
-	XMM_NUM extrd_opd2 \xmm
-	PFX_OPD_SIZE
-	PFX_REX extrd_opd1 extrd_opd2
-	.byte 0x0f, 0x3a, 0x16
-	MODRM 0xc0 extrd_opd1 extrd_opd2
-	.byte \imm8
-	.endm
-
-	.macro MOVQ_R64_XMM opd1 opd2
-	TYPE movq_r64_xmm_opd1_type \opd1
-	.if movq_r64_xmm_opd1_type == TYPE_XMM
-	XMM_NUM movq_r64_xmm_opd1 \opd1
-	R64_NUM movq_r64_xmm_opd2 \opd2
-	.else
-	R64_NUM movq_r64_xmm_opd1 \opd1
-	XMM_NUM movq_r64_xmm_opd2 \opd2
-	.endif
-	PFX_OPD_SIZE
-	PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
-	.if movq_r64_xmm_opd1_type == TYPE_XMM
-	.byte 0x0f, 0x7e
-	.else
-	.byte 0x0f, 0x6e
-	.endif
-	MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
-	.endm
-
-.text
-
-# Virtual Registers
-# ARG1
-msg =		%rdi
-# ARG2
-digest =	%rsi
-# ARG3
-msglen =	%rdx
-T1 =		%rcx
-T2 =		%r8
-a_64 =		%r9
-b_64 =		%r10
-c_64 =		%r11
-d_64 =		%r12
-e_64 =		%r13
-f_64 =		%r14
-g_64 =		%r15
-h_64 =		%rbx
-tmp0 =		%rax
-
-# Local variables (stack frame)
-
-W_SIZE = 80*8
-WK_SIZE = 2*8
-RSPSAVE_SIZE = 1*8
-GPRSAVE_SIZE = 5*8
-
-frame_W = 0
-frame_WK = frame_W + W_SIZE
-frame_RSPSAVE = frame_WK + WK_SIZE
-frame_GPRSAVE = frame_RSPSAVE + RSPSAVE_SIZE
-frame_size = frame_GPRSAVE + GPRSAVE_SIZE
-
-# Useful QWORD "arrays" for simpler memory references
-# MSG, DIGEST, K_t, W_t are arrays
-# WK_2(t) points to 1 of 2 qwords at frame.WK depdending on t being odd/even
-
-# Input message (arg1)
-#define MSG(i)    8*i(msg)
-
-# Output Digest (arg2)
-#define DIGEST(i) 8*i(digest)
-
-# SHA Constants (static mem)
-#define K_t(i)    8*i+K512(%rip)
-
-# Message Schedule (stack frame)
-#define W_t(i)    8*i+frame_W(%rsp)
-
-# W[t]+K[t] (stack frame)
-#define WK_2(i)   8*((i%2))+frame_WK(%rsp)
-
-.macro RotateState
-	# Rotate symbols a..h right
-	TMP   = h_64
-	h_64  = g_64
-	g_64  = f_64
-	f_64  = e_64
-	e_64  = d_64
-	d_64  = c_64
-	c_64  = b_64
-	b_64  = a_64
-	a_64  = TMP
-.endm
-
-.macro SHA512_Round rnd
-
-	# Compute Round %%t
-	mov	f_64, T1          # T1 = f
-	mov	e_64, tmp0        # tmp = e
-	xor	g_64, T1          # T1 = f ^ g
-	ror	$23, tmp0 # 41    # tmp = e ror 23
-	and	e_64, T1          # T1 = (f ^ g) & e
-	xor	e_64, tmp0        # tmp = (e ror 23) ^ e
-	xor	g_64, T1          # T1 = ((f ^ g) & e) ^ g = CH(e,f,g)
-	idx = \rnd
-	add	WK_2(idx), T1     # W[t] + K[t] from message scheduler
-	ror	$4, tmp0  # 18    # tmp = ((e ror 23) ^ e) ror 4
-	xor	e_64, tmp0        # tmp = (((e ror 23) ^ e) ror 4) ^ e
-	mov	a_64, T2          # T2 = a
-	add	h_64, T1          # T1 = CH(e,f,g) + W[t] + K[t] + h
-	ror	$14, tmp0 # 14    # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e)
-	add	tmp0, T1          # T1 = CH(e,f,g) + W[t] + K[t] + S1(e)
-	mov	a_64, tmp0        # tmp = a
-	xor	c_64, T2          # T2 = a ^ c
-	and	c_64, tmp0        # tmp = a & c
-	and	b_64, T2          # T2 = (a ^ c) & b
-	xor	tmp0, T2          # T2 = ((a ^ c) & b) ^ (a & c) = Maj(a,b,c)
-	mov	a_64, tmp0        # tmp = a
-	ror	$5, tmp0 # 39     # tmp = a ror 5
-	xor	a_64, tmp0        # tmp = (a ror 5) ^ a
-	add	T1, d_64          # e(next_state) = d + T1
-	ror	$6, tmp0 # 34     # tmp = ((a ror 5) ^ a) ror 6
-	xor	a_64, tmp0        # tmp = (((a ror 5) ^ a) ror 6) ^ a
-	lea	(T1, T2), h_64    # a(next_state) = T1 + Maj(a,b,c)
-	ror	$28, tmp0 # 28    # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a)
-	add	tmp0, h_64        # a(next_state) = T1 + Maj(a,b,c) S0(a)
-	RotateState
-.endm
-
-.macro SHA512_2Sched_2Round_sse rnd
-
-	# Compute rounds t-2 and t-1
-	# Compute message schedule QWORDS t and t+1
-
-	#   Two rounds are computed based on the values for K[t-2]+W[t-2] and
-	# K[t-1]+W[t-1] which were previously stored at WK_2 by the message
-	# scheduler.
-	#   The two new schedule QWORDS are stored at [W_t(%%t)] and [W_t(%%t+1)].
-	# They are then added to their respective SHA512 constants at
-	# [K_t(%%t)] and [K_t(%%t+1)] and stored at dqword [WK_2(%%t)]
-	#   For brievity, the comments following vectored instructions only refer to
-	# the first of a pair of QWORDS.
-	# Eg. XMM2=W[t-2] really means XMM2={W[t-2]|W[t-1]}
-	#   The computation of the message schedule and the rounds are tightly
-	# stitched to take advantage of instruction-level parallelism.
-	# For clarity, integer instructions (for the rounds calculation) are indented
-	# by one tab. Vectored instructions (for the message scheduler) are indented
-	# by two tabs.
-
-	mov	f_64, T1
-	idx = \rnd -2
-	movdqa	W_t(idx), %xmm2		    # XMM2 = W[t-2]
-	xor	g_64, T1
-	and	e_64, T1
-	movdqa	%xmm2, %xmm0	            # XMM0 = W[t-2]
-	xor	g_64, T1
-	idx = \rnd
-	add	WK_2(idx), T1
-	idx = \rnd - 15
-	movdqu	W_t(idx), %xmm5		    # XMM5 = W[t-15]
-	mov	e_64, tmp0
-	ror	$23, tmp0 # 41
-	movdqa	%xmm5, %xmm3	            # XMM3 = W[t-15]
-	xor	e_64, tmp0
-	ror	$4, tmp0 # 18
-	psrlq	$61-19, %xmm0		    # XMM0 = W[t-2] >> 42
-	xor	e_64, tmp0
-	ror	$14, tmp0 # 14
-	psrlq	$(8-7), %xmm3		    # XMM3 = W[t-15] >> 1
-	add	tmp0, T1
-	add	h_64, T1
-	pxor	%xmm2, %xmm0                # XMM0 = (W[t-2] >> 42) ^ W[t-2]
-	mov	a_64, T2
-	xor	c_64, T2
-	pxor	%xmm5, %xmm3                # XMM3 = (W[t-15] >> 1) ^ W[t-15]
-	and	b_64, T2
-	mov	a_64, tmp0
-	psrlq	$(19-6), %xmm0		    # XMM0 = ((W[t-2]>>42)^W[t-2])>>13
-	and	c_64, tmp0
-	xor	tmp0, T2
-	psrlq	$(7-1), %xmm3		    # XMM3 = ((W[t-15]>>1)^W[t-15])>>6
-	mov	a_64, tmp0
-	ror	$5, tmp0 # 39
-	pxor	%xmm2, %xmm0	            # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
-	xor	a_64, tmp0
-	ror	$6, tmp0 # 34
-	pxor	%xmm5, %xmm3                # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
-	xor	a_64, tmp0
-	ror	$28, tmp0 # 28
-	psrlq	$6, %xmm0                   # XMM0 = ((((W[t-2]>>42)^W[t-2])>>13)^W[t-2])>>6
-	add	tmp0, T2
-	add	T1, d_64
-	psrlq	$1, %xmm3                   # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1
-	lea	(T1, T2), h_64
-	RotateState
-	movdqa	%xmm2, %xmm1	            # XMM1 = W[t-2]
-	mov	f_64, T1
-	xor	g_64, T1
-	movdqa	%xmm5, %xmm4		    # XMM4 = W[t-15]
-	and	e_64, T1
-	xor	g_64, T1
-	psllq	$(64-19)-(64-61) , %xmm1    # XMM1 = W[t-2] << 42
-	idx = \rnd + 1
-	add	WK_2(idx), T1
-	mov	e_64, tmp0
-	psllq	$(64-1)-(64-8), %xmm4	    # XMM4 = W[t-15] << 7
-	ror	$23, tmp0 # 41
-	xor	e_64, tmp0
-	pxor	%xmm2, %xmm1		    # XMM1 = (W[t-2] << 42)^W[t-2]
-	ror	$4, tmp0 # 18
-	xor	e_64, tmp0
-	pxor	%xmm5, %xmm4		    # XMM4 = (W[t-15]<<7)^W[t-15]
-	ror	$14, tmp0 # 14
-	add	tmp0, T1
-	psllq	$(64-61), %xmm1		    # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
-	add	h_64, T1
-	mov	a_64, T2
-	psllq	$(64-8), %xmm4		    # XMM4 = ((W[t-15]<<7)^W[t-15])<<56
-	xor	c_64, T2
-	and	b_64, T2
-	pxor	%xmm1, %xmm0		    # XMM0 = s1(W[t-2])
-	mov	a_64, tmp0
-	and	c_64, tmp0
-	idx = \rnd - 7
-	movdqu	W_t(idx), %xmm1		    # XMM1 = W[t-7]
-	xor	tmp0, T2
-	pxor	%xmm4, %xmm3                # XMM3 = s0(W[t-15])
-	mov	a_64, tmp0
-	paddq	%xmm3, %xmm0		    # XMM0 = s1(W[t-2]) + s0(W[t-15])
-	ror	$5, tmp0 # 39
-	idx =\rnd-16
-	paddq	W_t(idx), %xmm0		    # XMM0 = s1(W[t-2]) + s0(W[t-15]) + W[t-16]
-	xor	a_64, tmp0
-	paddq	%xmm1, %xmm0	            # XMM0 = s1(W[t-2]) + W[t-7] + s0(W[t-15]) + W[t-16]
-	ror	$6, tmp0 # 34
-	movdqa	%xmm0, W_t(\rnd)	    # Store scheduled qwords
-	xor	a_64, tmp0
-	paddq	K_t(\rnd), %xmm0	    # Compute W[t]+K[t]
-	ror	$28, tmp0 # 28
-	idx = \rnd
-	movdqa	%xmm0, WK_2(idx)	    # Store W[t]+K[t] for next rounds
-	add	tmp0, T2
-	add	T1, d_64
-	lea	(T1, T2), h_64
-	RotateState
-.endm
-
-########################################################################
-# void sha512_transform_ssse3(const void* M, void* D, u64 L)#
-# Purpose: Updates the SHA512 digest stored at D with the message stored in M.
-# The size of the message pointed to by M must be an integer multiple of SHA512
-#   message blocks.
-# L is the message length in SHA512 blocks.
-########################################################################
-ENTRY(sha512_transform_ssse3)
-
-	cmp $0, msglen
-	je nowork
-
-	# Allocate Stack Space
-	mov	%rsp, %rax
-	sub	$frame_size, %rsp
-	and	$~(0x20 - 1), %rsp
-	mov	%rax, frame_RSPSAVE(%rsp)
-
-	# Save GPRs
-	mov	%rbx, frame_GPRSAVE(%rsp)
-	mov	%r12, frame_GPRSAVE +8*1(%rsp)
-	mov	%r13, frame_GPRSAVE +8*2(%rsp)
-	mov	%r14, frame_GPRSAVE +8*3(%rsp)
-	mov	%r15, frame_GPRSAVE +8*4(%rsp)
-
-updateblock:
-
-# Load state variables
-	mov	DIGEST(0), a_64
-	mov	DIGEST(1), b_64
-	mov	DIGEST(2), c_64
-	mov	DIGEST(3), d_64
-	mov	DIGEST(4), e_64
-	mov	DIGEST(5), f_64
-	mov	DIGEST(6), g_64
-	mov	DIGEST(7), h_64
-
-	t = 0
-	.rept 80/2 + 1
-	# (80 rounds) / (2 rounds/iteration) + (1 iteration)
-	# +1 iteration because the scheduler leads hashing by 1 iteration
-		.if t < 2
-			# BSWAP 2 QWORDS
-			movdqa	XMM_QWORD_BSWAP(%rip), %xmm1
-			movdqu	MSG(t), %xmm0
-			pshufb	%xmm1, %xmm0	# BSWAP
-			movdqa	%xmm0, W_t(t)	# Store Scheduled Pair
-			paddq	K_t(t), %xmm0	# Compute W[t]+K[t]
-			movdqa	%xmm0, WK_2(t)	# Store into WK for rounds
-		.elseif t < 16
-			# BSWAP 2 QWORDS# Compute 2 Rounds
-			movdqu	MSG(t), %xmm0
-			pshufb	%xmm1, %xmm0	# BSWAP
-			SHA512_Round t-2	# Round t-2
-			movdqa	%xmm0, W_t(t)	# Store Scheduled Pair
-			paddq	K_t(t), %xmm0	# Compute W[t]+K[t]
-			SHA512_Round t-1	# Round t-1
-			movdqa	%xmm0, WK_2(t)	# Store W[t]+K[t] into WK
-		.elseif t < 79
-			# Schedule 2 QWORDS# Compute 2 Rounds
-			SHA512_2Sched_2Round_sse t
-		.else
-			# Compute 2 Rounds
-			SHA512_Round t-2
-			SHA512_Round t-1
-		.endif
-		t = t+2
-	.endr
-
-	# Update digest
-	add	a_64, DIGEST(0)
-	add	b_64, DIGEST(1)
-	add	c_64, DIGEST(2)
-	add	d_64, DIGEST(3)
-	add	e_64, DIGEST(4)
-	add	f_64, DIGEST(5)
-	add	g_64, DIGEST(6)
-	add	h_64, DIGEST(7)
-
-	# Advance to next message block
-	add	$16*8, msg
-	dec	msglen
-	jnz	updateblock
-
-	# Restore GPRs
-	mov	frame_GPRSAVE(%rsp),      %rbx
-	mov	frame_GPRSAVE +8*1(%rsp), %r12
-	mov	frame_GPRSAVE +8*2(%rsp), %r13
-	mov	frame_GPRSAVE +8*3(%rsp), %r14
-	mov	frame_GPRSAVE +8*4(%rsp), %r15
-
-	# Restore Stack Pointer
-	mov	frame_RSPSAVE(%rsp), %rsp
-
-nowork:
-	ret
-ENDPROC(sha512_transform_ssse3)
-
-########################################################################
-### Binary Data
-
-.data
-
-.align 16
-
-# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
-XMM_QWORD_BSWAP:
-	.octa 0x08090a0b0c0d0e0f0001020304050607
-
-# K[t] used in SHA512 hashing
-K512:
-	.quad 0x428a2f98d728ae22,0x7137449123ef65cd
-	.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
-	.quad 0x3956c25bf348b538,0x59f111f1b605d019
-	.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
-	.quad 0xd807aa98a3030242,0x12835b0145706fbe
-	.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
-	.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
-	.quad 0x9bdc06a725c71235,0xc19bf174cf692694
-	.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
-	.quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
-	.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
-	.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
-	.quad 0x983e5152ee66dfab,0xa831c66d2db43210
-	.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
-	.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
-	.quad 0x06ca6351e003826f,0x142929670a0e6e70
-	.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
-	.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
-	.quad 0x650a73548baf63de,0x766a0abb3c77b2a8
-	.quad 0x81c2c92e47edaee6,0x92722c851482353b
-	.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
-	.quad 0xc24b8b70d0f89791,0xc76c51a30654be30
-	.quad 0xd192e819d6ef5218,0xd69906245565a910
-	.quad 0xf40e35855771202a,0x106aa07032bbd1b8
-	.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
-	.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
-	.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
-	.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
-	.quad 0x748f82ee5defb2fc,0x78a5636f43172f60
-	.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
-	.quad 0x90befffa23631e28,0xa4506cebde82bde9
-	.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
-	.quad 0xca273eceea26619c,0xd186b8c721c0c207
-	.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
-	.quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
-	.quad 0x113f9804bef90dae,0x1b710b35131c471b
-	.quad 0x28db77f523047d84,0x32caab7b40c72493
-	.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
-	.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
-	.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
-#endif

pkg/crypto/sha512/sha512_linux.go

@@ -1,166 +0,0 @@
-// +build linux,amd64,cgo
-
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file of
-// Golang project:
-//    https://github.com/golang/go/blob/master/LICENSE
-
-// Using this part of Minio codebase under the license
-// Apache License Version 2.0 with modifications
-
-// Package sha512 implements the SHA512 hash algorithms as defined
-// in FIPS 180-2.
-package sha512
-
-import (
-	"hash"
-
-	"github.com/klauspost/cpuid"
-)
-
-// Size - The size of a SHA512 checksum in bytes.
-const Size = 64
-
-// BlockSize - The blocksize of SHA512 in bytes.
-const BlockSize = 128
-
-const (
-	chunk = 128
-	init0 = 0x6a09e667f3bcc908
-	init1 = 0xbb67ae8584caa73b
-	init2 = 0x3c6ef372fe94f82b
-	init3 = 0xa54ff53a5f1d36f1
-	init4 = 0x510e527fade682d1
-	init5 = 0x9b05688c2b3e6c1f
-	init6 = 0x1f83d9abfb41bd6b
-	init7 = 0x5be0cd19137e2179
-)
-
-// digest represents the partial evaluation of a checksum.
-type digest struct {
-	h   [8]uint64
-	x   [chunk]byte
-	nx  int
-	len uint64
-}
-
-func block(dig *digest, p []byte) {
-	switch true {
-	case cpuid.CPU.AVX2():
-		blockAVX2(dig, p)
-	case cpuid.CPU.AVX():
-		blockAVX(dig, p)
-	case cpuid.CPU.SSSE3():
-		blockSSE(dig, p)
-	default:
-		blockGeneric(dig, p)
-	}
-}
-
-// Reset digest to its default value
-func (d *digest) Reset() {
-	d.h[0] = init0
-	d.h[1] = init1
-	d.h[2] = init2
-	d.h[3] = init3
-	d.h[4] = init4
-	d.h[5] = init5
-	d.h[6] = init6
-	d.h[7] = init7
-	d.nx = 0
-	d.len = 0
-}
-
-// New returns a new hash.Hash computing the SHA512 checksum.
-func New() hash.Hash {
-	d := new(digest)
-	d.Reset()
-	return d
-}
-
-// Sum512 - single caller sha512 helper
-func Sum512(data []byte) [Size]byte {
-	var d digest
-	d.Reset()
-	d.Write(data)
-	return d.checkSum()
-}
-
-// Return output array byte size
-func (d *digest) Size() int { return Size }
-
-// Return blockSize
-func (d *digest) BlockSize() int { return BlockSize }
-
-// Write blocks
-func (d *digest) Write(p []byte) (nn int, err error) {
-	nn = len(p)
-	d.len += uint64(nn)
-	if d.nx > 0 {
-		n := copy(d.x[d.nx:], p)
-		d.nx += n
-		if d.nx == chunk {
-			block(d, d.x[:])
-			d.nx = 0
-		}
-		p = p[n:]
-	}
-	if len(p) >= chunk {
-		n := len(p) &^ (chunk - 1)
-		block(d, p[:n])
-		p = p[n:]
-	}
-	if len(p) > 0 {
-		d.nx = copy(d.x[:], p)
-	}
-	return
-}
-
-// Calculate sha512
-func (d *digest) Sum(in []byte) []byte {
-	// Make a copy of d0 so that caller can keep writing and summing.
-	d0 := *d
-	hash := d0.checkSum()
-	return append(in, hash[:]...)
-}
-
-// internal checksum calculation, returns [Size]byte
-func (d *digest) checkSum() [Size]byte {
-	// Padding.  Add a 1 bit and 0 bits until 112 bytes mod 128.
-	len := d.len
-	var tmp [128]byte
-	tmp[0] = 0x80
-	if len%128 < 112 {
-		d.Write(tmp[0 : 112-len%128])
-	} else {
-		d.Write(tmp[0 : 128+112-len%128])
-	}
-
-	// Length in bits.
-	len <<= 3
-	for i := uint(0); i < 16; i++ {
-		tmp[i] = byte(len >> (120 - 8*i))
-	}
-	d.Write(tmp[0:16])
-
-	if d.nx != 0 {
-		panic("d.nx != 0")
-	}
-
-	h := d.h[:]
-
-	var digest [Size]byte
-	for i, s := range h {
-		digest[i*8] = byte(s >> 56)
-		digest[i*8+1] = byte(s >> 48)
-		digest[i*8+2] = byte(s >> 40)
-		digest[i*8+3] = byte(s >> 32)
-		digest[i*8+4] = byte(s >> 24)
-		digest[i*8+5] = byte(s >> 16)
-		digest[i*8+6] = byte(s >> 8)
-		digest[i*8+7] = byte(s)
-	}
-
-	return digest
-}

pkg/crypto/sha512/sha512_test.go

@@ -1,141 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file of
-// Golang project:
-//    https://github.com/golang/go/blob/master/LICENSE
-
-// Using this part of Minio codebase under the license
-// Apache License Version 2.0 with modifications
-
-// SHA512 hash algorithm.  See FIPS 180-2.
-
-package sha512
-
-import (
-	"fmt"
-	"io"
-	"testing"
-)
-
-type sha512Test struct {
-	out string
-	in  string
-}
-
-var golden = []sha512Test{
-	{"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", ""},
-	{"1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75", "a"},
-	{"2d408a0717ec188158278a796c689044361dc6fdde28d6f04973b80896e1823975cdbf12eb63f9e0591328ee235d80e9b5bf1aa6a44f4617ff3caf6400eb172d", "ab"},
-	{"ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", "abc"},
-	{"d8022f2060ad6efd297ab73dcc5355c9b214054b0d1776a136a669d26a7d3b14f73aa0d0ebff19ee333368f0164b6419a96da49e3e481753e7e96b716bdccb6f", "abcd"},
-	{"878ae65a92e86cac011a570d4c30a7eaec442b85ce8eca0c2952b5e3cc0628c2e79d889ad4d5c7c626986d452dd86374b6ffaa7cd8b67665bef2289a5c70b0a1", "abcde"},
-	{"e32ef19623e8ed9d267f657a81944b3d07adbb768518068e88435745564e8d4150a0a703be2a7d88b61e3d390c2bb97e2d4c311fdc69d6b1267f05f59aa920e7", "abcdef"},
-	{"d716a4188569b68ab1b6dfac178e570114cdf0ea3a1cc0e31486c3e41241bc6a76424e8c37ab26f096fc85ef9886c8cb634187f4fddff645fb099f1ff54c6b8c", "abcdefg"},
-	{"a3a8c81bc97c2560010d7389bc88aac974a104e0e2381220c6e084c4dccd1d2d17d4f86db31c2a851dc80e6681d74733c55dcd03dd96f6062cdda12a291ae6ce", "abcdefgh"},
-	{"f22d51d25292ca1d0f68f69aedc7897019308cc9db46efb75a03dd494fc7f126c010e8ade6a00a0c1a5f1b75d81e0ed5a93ce98dc9b833db7839247b1d9c24fe", "abcdefghi"},
-	{"ef6b97321f34b1fea2169a7db9e1960b471aa13302a988087357c520be957ca119c3ba68e6b4982c019ec89de3865ccf6a3cda1fe11e59f98d99f1502c8b9745", "abcdefghij"},
-	{"2210d99af9c8bdecda1b4beff822136753d8342505ddce37f1314e2cdbb488c6016bdaa9bd2ffa513dd5de2e4b50f031393d8ab61f773b0e0130d7381e0f8a1d", "Discard medicine more than two years old."},
-	{"a687a8985b4d8d0a24f115fe272255c6afaf3909225838546159c1ed685c211a203796ae8ecc4c81a5b6315919b3a64f10713da07e341fcdbb08541bf03066ce", "He who has a shady past knows that nice guys finish last."},
-	{"8ddb0392e818b7d585ab22769a50df660d9f6d559cca3afc5691b8ca91b8451374e42bcdabd64589ed7c91d85f626596228a5c8572677eb98bc6b624befb7af8", "I wouldn't marry him with a ten foot pole."},
-	{"26ed8f6ca7f8d44b6a8a54ae39640fa8ad5c673f70ee9ce074ba4ef0d483eea00bab2f61d8695d6b34df9c6c48ae36246362200ed820448bdc03a720366a87c6", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
-	{"e5a14bf044be69615aade89afcf1ab0389d5fc302a884d403579d1386a2400c089b0dbb387ed0f463f9ee342f8244d5a38cfbc0e819da9529fbff78368c9a982", "The days of the digital watch are numbered.  -Tom Stoppard"},
-	{"420a1faa48919e14651bed45725abe0f7a58e0f099424c4e5a49194946e38b46c1f8034b18ef169b2e31050d1648e0b982386595f7df47da4b6fd18e55333015", "Nepal premier won't resign."},
-	{"d926a863beadb20134db07683535c72007b0e695045876254f341ddcccde132a908c5af57baa6a6a9c63e6649bba0c213dc05fadcf9abccea09f23dcfb637fbe", "For every action there is an equal and opposite government program."},
-	{"9a98dd9bb67d0da7bf83da5313dff4fd60a4bac0094f1b05633690ffa7f6d61de9a1d4f8617937d560833a9aaa9ccafe3fd24db418d0e728833545cadd3ad92d", "His money is twice tainted: 'taint yours and 'taint mine."},
-	{"d7fde2d2351efade52f4211d3746a0780a26eec3df9b2ed575368a8a1c09ec452402293a8ea4eceb5a4f60064ea29b13cdd86918cd7a4faf366160b009804107", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
-	{"b0f35ffa2697359c33a56f5c0cf715c7aeed96da9905ca2698acadb08fbc9e669bf566b6bd5d61a3e86dc22999bcc9f2224e33d1d4f32a228cf9d0349e2db518", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
-	{"3d2e5f91778c9e66f7e061293aaa8a8fc742dd3b2e4f483772464b1144189b49273e610e5cccd7a81a19ca1fa70f16b10f1a100a4d8c1372336be8484c64b311", "size:  a.out:  bad magic"},
-	{"b2f68ff58ac015efb1c94c908b0d8c2bf06f491e4de8e6302c49016f7f8a33eac3e959856c7fddbc464de618701338a4b46f76dbfaf9a1e5262b5f40639771c7", "The major problem is with sendmail.  -Mark Horton"},
-	{"d8c92db5fdf52cf8215e4df3b4909d29203ff4d00e9ad0b64a6a4e04dec5e74f62e7c35c7fb881bd5de95442123df8f57a489b0ae616bd326f84d10021121c57", "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
-	{"19a9f8dc0a233e464e8566ad3ca9b91e459a7b8c4780985b015776e1bf239a19bc233d0556343e2b0a9bc220900b4ebf4f8bdf89ff8efeaf79602d6849e6f72e", "If the enemy is within range, then so are you."},
-	{"00b4c41f307bde87301cdc5b5ab1ae9a592e8ecbb2021dd7bc4b34e2ace60741cc362560bec566ba35178595a91932b8d5357e2c9cec92d393b0fa7831852476", "It's well we cannot hear the screams/That we create in others' dreams."},
-	{"91eccc3d5375fd026e4d6787874b1dce201cecd8a27dbded5065728cb2d09c58a3d467bb1faf353bf7ba567e005245d5321b55bc344f7c07b91cb6f26c959be7", "You remind me of a TV show, but that's all right: I watch it anyway."},
-	{"fabbbe22180f1f137cfdc9556d2570e775d1ae02a597ded43a72a40f9b485d500043b7be128fb9fcd982b83159a0d99aa855a9e7cc4240c00dc01a9bdf8218d7", "C is as portable as Stonehedge!!"},
-	{"2ecdec235c1fa4fc2a154d8fba1dddb8a72a1ad73838b51d792331d143f8b96a9f6fcb0f34d7caa351fe6d88771c4f105040e0392f06e0621689d33b2f3ba92e", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
-	{"7ad681f6f96f82f7abfa7ecc0334e8fa16d3dc1cdc45b60b7af43fe4075d2357c0c1d60e98350f1afb1f2fe7a4d7cd2ad55b88e458e06b73c40b437331f5dab4", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
-	{"833f9248ab4a3b9e5131f745fda1ffd2dd435b30e965957e78291c7ab73605fd1912b0794e5c233ab0a12d205a39778d19b83515d6a47003f19cdee51d98c7e0", "How can you write a big system without C++?  -Paul Glick"},
-}
-
-func TestGolden(t *testing.T) {
-	for i := 0; i < len(golden); i++ {
-		g := golden[i]
-		s := fmt.Sprintf("%x", Sum512([]byte(g.in)))
-		if s != g.out {
-			t.Fatalf("Sum512 function: sha512(%s) = %s want %s", g.in, s, g.out)
-		}
-		c := New()
-		for j := 0; j < 3; j++ {
-			if j < 2 {
-				io.WriteString(c, g.in)
-			} else {
-				io.WriteString(c, g.in[0:len(g.in)/2])
-				c.Sum(nil)
-				io.WriteString(c, g.in[len(g.in)/2:])
-			}
-			s := fmt.Sprintf("%x", c.Sum(nil))
-			if s != g.out {
-				t.Fatalf("sha512[%d](%s) = %s want %s", j, g.in, s, g.out)
-			}
-			c.Reset()
-		}
-	}
-}
-
-func TestSize(t *testing.T) {
-	c := New()
-	if got := c.Size(); got != Size {
-		t.Errorf("Size = %d; want %d", got, Size)
-	}
-}
-
-func TestBlockSize(t *testing.T) {
-	c := New()
-	if got := c.BlockSize(); got != BlockSize {
-		t.Errorf("BlockSize = %d; want %d", got, BlockSize)
-	}
-}
-
-var bench = New()
-var buf = make([]byte, 1024*1024)
-
-func benchmarkSize(b *testing.B, size int) {
-	b.SetBytes(int64(size))
-	sum := make([]byte, bench.Size())
-	for i := 0; i < b.N; i++ {
-		bench.Reset()
-		bench.Write(buf[:size])
-		bench.Sum(sum[:0])
-	}
-}
-
-func BenchmarkHash8Bytes(b *testing.B) {
-	benchmarkSize(b, 8)
-}
-
-func BenchmarkHash1K(b *testing.B) {
-	benchmarkSize(b, 1024)
-}
-
-func BenchmarkHash8K(b *testing.B) {
-	benchmarkSize(b, 8192)
-}
-
-func BenchmarkHash1M(b *testing.B) {
-	benchmarkSize(b, 1024*1024)
-}

pkg/crypto/sha512/sha512block.go

@@ -1,181 +0,0 @@
-// +build linux,amd64,cgo
-
-/*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Software block transform are provided by The Go Authors:
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file of
-// Golang project:
-//    https://github.com/golang/go/blob/master/LICENSE
-
-package sha512
-
-// #cgo CFLAGS: -DHAS_SSE41 -DHAS_AVX -DHAS_AVX2
-// #include <stdint.h>
-// void sha512_transform_ssse3 (const void* M, void* D, uint64_t L);
-// void sha512_transform_avx (const void* M, void* D, uint64_t L);
-// void sha512_transform_rorx (const void* M, void* D, uint64_t L);
-import "C"
-import (
-	"unsafe"
-)
-
-func blockSSE(dig *digest, p []byte) {
-	C.sha512_transform_ssse3(unsafe.Pointer(&p[0]), unsafe.Pointer(&dig.h[0]), (C.uint64_t)(len(p)/chunk))
-}
-
-func blockAVX(dig *digest, p []byte) {
-	C.sha512_transform_avx(unsafe.Pointer(&p[0]), unsafe.Pointer(&dig.h[0]), (C.uint64_t)(len(p)/chunk))
-}
-
-func blockAVX2(dig *digest, p []byte) {
-	C.sha512_transform_rorx(unsafe.Pointer(&p[0]), unsafe.Pointer(&dig.h[0]), (C.uint64_t)(len(p)/chunk))
-}
-
-func blockGeneric(dig *digest, p []byte) {
-	var w [80]uint64
-	h0, h1, h2, h3, h4, h5, h6, h7 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7]
-	for len(p) >= chunk {
-		for i := 0; i < 16; i++ {
-			j := i * 8
-			w[i] = uint64(p[j])<<56 | uint64(p[j+1])<<48 | uint64(p[j+2])<<40 | uint64(p[j+3])<<32 |
-				uint64(p[j+4])<<24 | uint64(p[j+5])<<16 | uint64(p[j+6])<<8 | uint64(p[j+7])
-		}
-		for i := 16; i < 80; i++ {
-			v1 := w[i-2]
-			t1 := (v1>>19 | v1<<(64-19)) ^ (v1>>61 | v1<<(64-61)) ^ (v1 >> 6)
-			v2 := w[i-15]
-			t2 := (v2>>1 | v2<<(64-1)) ^ (v2>>8 | v2<<(64-8)) ^ (v2 >> 7)
-
-			w[i] = t1 + w[i-7] + t2 + w[i-16]
-		}
-
-		a, b, c, d, e, f, g, h := h0, h1, h2, h3, h4, h5, h6, h7
-
-		for i := 0; i < 80; i++ {
-			t1 := h + ((e>>14 | e<<(64-14)) ^ (e>>18 | e<<(64-18)) ^ (e>>41 | e<<(64-41))) + ((e & f) ^ (^e & g)) + _K[i] + w[i]
-
-			t2 := ((a>>28 | a<<(64-28)) ^ (a>>34 | a<<(64-34)) ^ (a>>39 | a<<(64-39))) + ((a & b) ^ (a & c) ^ (b & c))
-
-			h = g
-			g = f
-			f = e
-			e = d + t1
-			d = c
-			c = b
-			b = a
-			a = t1 + t2
-		}
-
-		h0 += a
-		h1 += b
-		h2 += c
-		h3 += d
-		h4 += e
-		h5 += f
-		h6 += g
-		h7 += h
-
-		p = p[chunk:]
-	}
-
-	dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] = h0, h1, h2, h3, h4, h5, h6, h7
-}
-
-var _K = []uint64{
-	0x428a2f98d728ae22,
-	0x7137449123ef65cd,
-	0xb5c0fbcfec4d3b2f,
-	0xe9b5dba58189dbbc,
-	0x3956c25bf348b538,
-	0x59f111f1b605d019,
-	0x923f82a4af194f9b,
-	0xab1c5ed5da6d8118,
-	0xd807aa98a3030242,
-	0x12835b0145706fbe,
-	0x243185be4ee4b28c,
-	0x550c7dc3d5ffb4e2,
-	0x72be5d74f27b896f,
-	0x80deb1fe3b1696b1,
-	0x9bdc06a725c71235,
-	0xc19bf174cf692694,
-	0xe49b69c19ef14ad2,
-	0xefbe4786384f25e3,
-	0x0fc19dc68b8cd5b5,
-	0x240ca1cc77ac9c65,
-	0x2de92c6f592b0275,
-	0x4a7484aa6ea6e483,
-	0x5cb0a9dcbd41fbd4,
-	0x76f988da831153b5,
-	0x983e5152ee66dfab,
-	0xa831c66d2db43210,
-	0xb00327c898fb213f,
-	0xbf597fc7beef0ee4,
-	0xc6e00bf33da88fc2,
-	0xd5a79147930aa725,
-	0x06ca6351e003826f,
-	0x142929670a0e6e70,
-	0x27b70a8546d22ffc,
-	0x2e1b21385c26c926,
-	0x4d2c6dfc5ac42aed,
-	0x53380d139d95b3df,
-	0x650a73548baf63de,
-	0x766a0abb3c77b2a8,
-	0x81c2c92e47edaee6,
-	0x92722c851482353b,
-	0xa2bfe8a14cf10364,
-	0xa81a664bbc423001,
-	0xc24b8b70d0f89791,
-	0xc76c51a30654be30,
-	0xd192e819d6ef5218,
-	0xd69906245565a910,
-	0xf40e35855771202a,
-	0x106aa07032bbd1b8,
-	0x19a4c116b8d2d0c8,
-	0x1e376c085141ab53,
-	0x2748774cdf8eeb99,
-	0x34b0bcb5e19b48a8,
-	0x391c0cb3c5c95a63,
-	0x4ed8aa4ae3418acb,
-	0x5b9cca4f7763e373,
-	0x682e6ff3d6b2b8a3,
-	0x748f82ee5defb2fc,
-	0x78a5636f43172f60,
-	0x84c87814a1f0ab72,
-	0x8cc702081a6439ec,
-	0x90befffa23631e28,
-	0xa4506cebde82bde9,
-	0xbef9a3f7b2c67915,
-	0xc67178f2e372532b,
-	0xca273eceea26619c,
-	0xd186b8c721c0c207,
-	0xeada7dd6cde0eb1e,
-	0xf57d4f7fee6ed178,
-	0x06f067aa72176fba,
-	0x0a637dc5a2c898a6,
-	0x113f9804bef90dae,
-	0x1b710b35131c471b,
-	0x28db77f523047d84,
-	0x32caab7b40c72493,
-	0x3c9ebe0a15c9bebc,
-	0x431d67c49c100d4c,
-	0x4cc5d4becb3e42b6,
-	0x597f299cfc657e2a,
-	0x5fcb6fab3ad6faec,
-	0x6c44198c4a475817,
-}

pkg/disk/disk.go

@@ -23,5 +23,7 @@ package disk
 type Info struct {
 	Total  int64
 	Free   int64
+	Files  int64
+	Ffree  int64
 	FSType string
 }

pkg/disk/disk_test.go

@@ -41,5 +41,7 @@ func (s *MySuite) TestFree(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(di.Total, Not(Equals), 0)
 	c.Assert(di.Free, Not(Equals), 0)
+	c.Assert(di.Files, Not(Equals), 0)
+	c.Assert(di.Ffree, Not(Equals), 0)
 	c.Assert(di.FSType, Not(Equals), "UNKNOWN")
 }

pkg/disk/stat_nix.go

@@ -32,6 +32,8 @@ func GetInfo(path string) (info Info, err error) {
 	info = Info{}
 	info.Total = int64(s.Bsize) * int64(s.Blocks)
 	info.Free = int64(s.Bsize) * int64(s.Bfree)
+	info.Files = int64(s.Files)
+	info.Ffree = int64(s.Ffree)
 	info.FSType, err = getFSType(path)
 	if err != nil {
 		return Info{}, err

pkg/disk/stat_windows.go

@@ -19,6 +19,7 @@
 package disk
 
 import (
+	"os"
 	"syscall"
 	"unsafe"
 )
@@ -28,6 +29,11 @@ import (
 //
 // https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
 func GetInfo(path string) (info Info, err error) {
+	// Stat to know if the path exists.
+	if _, err = os.Stat(path); err != nil {
+		return Info{}, err
+	}
+
 	dll := syscall.MustLoadDLL("kernel32.dll")
 	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
 	// Retrieves information about the amount of space that is available on a disk volume,
@@ -54,5 +60,33 @@ func GetInfo(path string) (info Info, err error) {
 	info.Total = int64(lpTotalNumberOfBytes)
 	info.Free = int64(lpFreeBytesAvailable)
 	info.FSType = getFSType(path)
+
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364935(v=vs.85).aspx
+	// Retrieves information about the specified disk, including the amount of free space on the disk.
+	GetDiskFreeSpace := dll.MustFindProc("GetDiskFreeSpaceW")
+
+	// Return values of GetDiskFreeSpace()
+	lpSectorsPerCluster := uint32(0)
+	lpBytesPerSector := uint32(0)
+	lpNumberOfFreeClusters := uint32(0)
+	lpTotalNumberOfClusters := uint32(0)
+
+	// Extract values safely
+	// BOOL WINAPI GetDiskFreeSpace(
+	//   _In_  LPCTSTR lpRootPathName,
+	//   _Out_ LPDWORD lpSectorsPerCluster,
+	//   _Out_ LPDWORD lpBytesPerSector,
+	//   _Out_ LPDWORD lpNumberOfFreeClusters,
+	//   _Out_ LPDWORD lpTotalNumberOfClusters
+	// );
+	_, _, _ = GetDiskFreeSpace.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
+		uintptr(unsafe.Pointer(&lpSectorsPerCluster)),
+		uintptr(unsafe.Pointer(&lpBytesPerSector)),
+		uintptr(unsafe.Pointer(&lpNumberOfFreeClusters)),
+		uintptr(unsafe.Pointer(&lpTotalNumberOfClusters)))
+
+	info.Files = int64(lpTotalNumberOfClusters)
+	info.Ffree = int64(lpNumberOfFreeClusters)
+
 	return info, nil
 }

pkg/objcache/README.md

@@ -0,0 +1,62 @@
+```
+PACKAGE DOCUMENTATION
+
+package objcache
+    import "github.com/minio/minio/pkg/objcache"
+
+    Package objcache implements in memory caching methods.
+
+VARIABLES
+
+var DefaultExpiry = time.Duration(72 * time.Hour) // 72hrs.
+
+    DefaultExpiry represents default time duration value when individual
+    entries will be expired.
+
+var ErrCacheFull = errors.New("Not enough space in cache")
+    ErrCacheFull - cache is full.
+
+var ErrKeyNotFoundInCache = errors.New("Key not found in cache")
+    ErrKeyNotFoundInCache - key not found in cache.
+
+var NoExpiry = time.Duration(0)
+    NoExpiry represents caches to be permanent and can only be deleted.
+
+TYPES
+
+type Cache struct {
+
+    // OnEviction - callback function for eviction
+    OnEviction func(key string)
+    // contains filtered or unexported fields
+}
+    Cache holds the required variables to compose an in memory cache system
+    which also provides expiring key mechanism and also maxSize.
+
+func New(maxSize uint64, expiry time.Duration) *Cache
+    New - Return a new cache with a given default expiry duration. If the
+    expiry duration is less than one (or NoExpiry), the items in the cache
+    never expire (by default), and must be deleted manually.
+
+func (c *Cache) Create(key string, size int64) (w io.WriteCloser, err error)
+    Create - validates if object size fits with in cache size limit and
+    returns a io.WriteCloser to which object contents can be written and
+    finally Close()'d. During Close() we checks if the amount of data
+    written is equal to the size of the object, in which case it saves the
+    contents to object cache.
+
+func (c *Cache) Delete(key string)
+    Delete - delete deletes an entry from the cache.
+
+func (c *Cache) DeleteExpired()
+    DeleteExpired - deletes all the expired entries from the cache.
+
+func (c *Cache) Open(key string) (io.ReadSeeker, error)
+    Open - open the in-memory file, returns an in memory read seeker.
+    returns an error ErrNotFoundInCache, if the key does not exist.
+
+func (c *Cache) StopExpiry()
+    StopExpiry sends a message to the expiry routine to stop expiring cached
+    entries. NOTE: once this is called, cached entries will not be expired
+    if the consume has called this.
+```

pkg/objcache/objcache.go

@@ -0,0 +1,240 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package objcache implements in memory caching methods.
+package objcache
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"sync"
+	"time"
+)
+
+// NoExpiry represents caches to be permanent and can only be deleted.
+var NoExpiry = time.Duration(0)
+
+// DefaultExpiry represents default time duration value when individual entries will be expired.
+var DefaultExpiry = time.Duration(72 * time.Hour) // 72hrs.
+
+// buffer represents the in memory cache of a single entry.
+// buffer carries value of the data and last accessed time.
+type buffer struct {
+	value        []byte    // Value of the entry.
+	lastAccessed time.Time // Represents time when value was last accessed.
+}
+
+// Cache holds the required variables to compose an in memory cache system
+// which also provides expiring key mechanism and also maxSize.
+type Cache struct {
+	// Mutex is used for handling the concurrent
+	// read/write requests for cache
+	mutex sync.Mutex
+
+	// maxSize is a total size for overall cache
+	maxSize uint64
+
+	// currentSize is a current size in memory
+	currentSize uint64
+
+	// OnEviction - callback function for eviction
+	OnEviction func(key string)
+
+	// totalEvicted counter to keep track of total expirys
+	totalEvicted int
+
+	// map of objectName and its contents
+	entries map[string]*buffer
+
+	// Expiry in time duration.
+	expiry time.Duration
+
+	// Stop garbage collection routine, stops any running GC routine.
+	stopGC chan struct{}
+}
+
+// New - Return a new cache with a given default expiry duration.
+// If the expiry duration is less than one (or NoExpiry),
+// the items in the cache never expire (by default), and must be deleted
+// manually.
+func New(maxSize uint64, expiry time.Duration) *Cache {
+	C := &Cache{
+		maxSize: maxSize,
+		entries: make(map[string]*buffer),
+		expiry:  expiry,
+	}
+	// We have expiry start the janitor routine.
+	if expiry > 0 {
+		C.stopGC = make(chan struct{})
+
+		// Start garbage collection routine to expire objects.
+		C.startGC()
+	}
+	return C
+}
+
+// ErrKeyNotFoundInCache - key not found in cache.
+var ErrKeyNotFoundInCache = errors.New("Key not found in cache")
+
+// ErrCacheFull - cache is full.
+var ErrCacheFull = errors.New("Not enough space in cache")
+
+// ErrExcessData - excess data was attempted to be written on cache.
+var ErrExcessData = errors.New("Attempted excess write on cache")
+
+// Used for adding entry to the object cache. Implements io.WriteCloser
+type cacheBuffer struct {
+	*bytes.Buffer // Implements io.Writer
+	onClose       func() error
+}
+
+// On close, onClose() is called which checks if all object contents
+// have been written so that it can save the buffer to the cache.
+func (c cacheBuffer) Close() (err error) {
+	return c.onClose()
+}
+
+// Create - validates if object size fits with in cache size limit and returns a io.WriteCloser
+// to which object contents can be written and finally Close()'d. During Close() we
+// checks if the amount of data written is equal to the size of the object, in which
+// case it saves the contents to object cache.
+func (c *Cache) Create(key string, size int64) (w io.WriteCloser, err error) {
+	// Recovers any panic generated and return errors appropriately.
+	defer func() {
+		if r := recover(); r != nil {
+			// Recover any panic and return ErrCacheFull.
+			err = ErrCacheFull
+		}
+	}() // Do not crash the server.
+
+	valueLen := uint64(size)
+	// Check if the size of the object is not bigger than the capacity of the cache.
+	if c.maxSize > 0 && valueLen > c.maxSize {
+		return nil, ErrCacheFull
+	}
+
+	// Will hold the object contents.
+	buf := bytes.NewBuffer(make([]byte, 0, size))
+
+	// Function called on close which saves the object contents
+	// to the object cache.
+	onClose := func() error {
+		c.mutex.Lock()
+		defer c.mutex.Unlock()
+		if size != int64(buf.Len()) {
+			// Full object not available hence do not save buf to object cache.
+			return io.ErrShortBuffer
+		}
+		if c.maxSize > 0 && c.currentSize+valueLen > c.maxSize {
+			return ErrExcessData
+		}
+		// Full object available in buf, save it to cache.
+		c.entries[key] = &buffer{
+			value:        buf.Bytes(),
+			lastAccessed: time.Now().UTC(), // Save last accessed time.
+		}
+		// Account for the memory allocated above.
+		c.currentSize += uint64(size)
+		return nil
+	}
+
+	// Object contents that is written - cacheBuffer.Write(data)
+	// will be accumulated in buf which implements io.Writer.
+	return cacheBuffer{
+		buf,
+		onClose,
+	}, nil
+}
+
+// Open - open the in-memory file, returns an in memory read seeker.
+// returns an error ErrNotFoundInCache, if the key does not exist.
+func (c *Cache) Open(key string) (io.ReadSeeker, error) {
+	// Entry exists, return the readable buffer.
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	buf, ok := c.entries[key]
+	if !ok {
+		return nil, ErrKeyNotFoundInCache
+	}
+	buf.lastAccessed = time.Now().UTC()
+	return bytes.NewReader(buf.value), nil
+}
+
+// Delete - delete deletes an entry from the cache.
+func (c *Cache) Delete(key string) {
+	c.mutex.Lock()
+	c.delete(key)
+	c.mutex.Unlock()
+	if c.OnEviction != nil {
+		c.OnEviction(key)
+	}
+}
+
+// gc - garbage collect all the expired entries from the cache.
+func (c *Cache) gc() {
+	var evictedEntries []string
+	c.mutex.Lock()
+	for k, v := range c.entries {
+		if c.expiry > 0 && time.Now().UTC().Sub(v.lastAccessed) > c.expiry {
+			c.delete(k)
+			evictedEntries = append(evictedEntries, k)
+		}
+	}
+	c.mutex.Unlock()
+	for _, k := range evictedEntries {
+		if c.OnEviction != nil {
+			c.OnEviction(k)
+		}
+	}
+}
+
+// StopGC sends a message to the expiry routine to stop
+// expiring cached entries. NOTE: once this is called, cached
+// entries will not be expired if the consumer has called this.
+func (c *Cache) StopGC() {
+	if c.stopGC != nil {
+		c.stopGC <- struct{}{}
+	}
+}
+
+// startGC starts running a routine ticking at expiry interval, on each interval
+// this routine does a sweep across the cache entries and garbage collects all the
+// expired entries.
+func (c *Cache) startGC() {
+	go func() {
+		for {
+			select {
+			// Wait till cleanup interval and initiate delete expired entries.
+			case <-time.After(c.expiry / 4):
+				c.gc()
+				// Stop the routine, usually called by the user of object cache during cleanup.
+			case <-c.stopGC:
+				return
+			}
+		}
+	}()
+}
+
+// Deletes a requested entry from the cache.
+func (c *Cache) delete(key string) {
+	if buf, ok := c.entries[key]; ok {
+		delete(c.entries, key)
+		c.currentSize -= uint64(len(buf.value))
+		c.totalEvicted++
+	}
+}

pkg/objcache/objcache_test.go

@@ -0,0 +1,172 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package objcache
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"testing"
+	"time"
+)
+
+// TestObjCache - tests various cases for object cache behavior.
+func TestObjCache(t *testing.T) {
+	// Non exhaustive list of all object cache behavior cases.
+	testCases := []struct {
+		expiry    time.Duration
+		cacheSize uint64
+		err       error
+		closeErr  error
+	}{
+		// Validate if a key is not found in cache and Open fails.
+		{
+			expiry:    NoExpiry,
+			cacheSize: 1024,
+			err:       ErrKeyNotFoundInCache,
+		},
+		// Validate if cache indicates that it is full and Create fails.
+		{
+			expiry:    NoExpiry,
+			cacheSize: 1,
+			err:       ErrCacheFull,
+		},
+		// Validate if Create succeeds but Close fails to write to buffer.
+		{
+			expiry:    NoExpiry,
+			cacheSize: 2,
+			closeErr:  io.ErrShortBuffer,
+		},
+		// Validate that Create and Close succeed, making sure to update the cache.
+		{
+			expiry:    NoExpiry,
+			cacheSize: 1024,
+		},
+		// Validate that Delete succeeds and Open fails with key not found in cache.
+		{
+			expiry:    NoExpiry,
+			cacheSize: 1024,
+			err:       ErrKeyNotFoundInCache,
+		},
+		// Validate OnEviction function is called upon entry delete.
+		{
+			expiry:    NoExpiry,
+			cacheSize: 1024,
+		},
+	}
+
+	// Test 1 validating Open failure.
+	testCase := testCases[0]
+	cache := New(testCase.cacheSize, testCase.expiry)
+	_, err := cache.Open("test")
+	if testCase.err != err {
+		t.Errorf("Test case 2 expected to pass, failed instead %s", err)
+	}
+
+	// Test 2 validating Create failure.
+	testCase = testCases[1]
+	cache = New(testCase.cacheSize, testCase.expiry)
+	_, err = cache.Create("test", 2)
+	if testCase.err != err {
+		t.Errorf("Test case 2 expected to pass, failed instead %s", err)
+	}
+
+	// Test 3 validating Create succeeds and returns a writer.
+	// Subsequently we Close() without writing any data, to receive
+	// `io.ErrShortBuffer`
+	testCase = testCases[2]
+	cache = New(testCase.cacheSize, testCase.expiry)
+	w, err := cache.Create("test", 1)
+	if testCase.err != err {
+		t.Errorf("Test case 3 expected to pass, failed instead %s", err)
+	}
+	if err = w.Close(); err != testCase.closeErr {
+		t.Errorf("Test case 3 expected to pass, failed instead %s", err)
+	}
+
+	// Test 4 validates Create and Close succeeds successfully caching
+	// the writes.
+	testCase = testCases[3]
+	cache = New(testCase.cacheSize, testCase.expiry)
+	w, err = cache.Create("test", 5)
+	if testCase.err != err {
+		t.Errorf("Test case 4 expected to pass, failed instead %s", err)
+	}
+	// Write '5' bytes.
+	w.Write([]byte("Hello"))
+	// Close to successfully save into cache.
+	if err = w.Close(); err != nil {
+		t.Errorf("Test case 4 expected to pass, failed instead %s", err)
+	}
+	r, err := cache.Open("test")
+	if err != nil {
+		t.Errorf("Test case 4 expected to pass, failed instead %s", err)
+	}
+	// Reads everything stored for key "test".
+	cbytes, err := ioutil.ReadAll(r)
+	if err != nil {
+		t.Errorf("Test case 4 expected to pass, failed instead %s", err)
+	}
+	// Validate if read bytes match.
+	if !bytes.Equal(cbytes, []byte("Hello")) {
+		t.Errorf("Test case 4 expected to pass. wanted \"Hello\", got %s", string(cbytes))
+	}
+
+	// Test 5 validates Delete succeeds and Open fails with err
+	testCase = testCases[4]
+	cache = New(testCase.cacheSize, testCase.expiry)
+	w, err = cache.Create("test", 5)
+	if err != nil {
+		t.Errorf("Test case 5 expected to pass, failed instead %s", err)
+	}
+	// Write '5' bytes.
+	w.Write([]byte("Hello"))
+	// Close to successfully save into cache.
+	if err = w.Close(); err != nil {
+		t.Errorf("Test case 5 expected to pass, failed instead %s", err)
+	}
+	// Delete the cache entry.
+	cache.Delete("test")
+	_, err = cache.Open("test")
+	if testCase.err != err {
+		t.Errorf("Test case 5 expected to pass, failed instead %s", err)
+	}
+
+	// Test 6 validates OnEviction being called upon Delete is being invoked.
+	testCase = testCases[5]
+	cache = New(testCase.cacheSize, testCase.expiry)
+	w, err = cache.Create("test", 5)
+	if err != nil {
+		t.Errorf("Test case 6 expected to pass, failed instead %s", err)
+	}
+	// Write '5' bytes.
+	w.Write([]byte("Hello"))
+	// Close to successfully save into cache.
+	if err = w.Close(); err != nil {
+		t.Errorf("Test case 6 expected to pass, failed instead %s", err)
+	}
+	var deleteKey string
+	cache.OnEviction = func(key string) {
+		deleteKey = key
+	}
+	// Delete the cache entry.
+	cache.Delete("test")
+	if deleteKey != "test" {
+		t.Errorf("Test case 6 expected to pass, wanted \"test\", got %s", deleteKey)
+	}
+}

pkg/quick/errorutil.go

@@ -26,7 +26,7 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/olekukonko/ts"
+	"github.com/cheggaaa/pb"
 )
 
 const errorFmt = "%5d: %s <--  "
@@ -53,8 +53,8 @@ func FormatJSONSyntaxError(data io.Reader, sErr *json.SyntaxError) error {
 	// dynamically to avoid an eventual bug after modifying errorFmt
 	errorShift := len(fmt.Sprintf(errorFmt, 1, ""))
 
-	if termSize, err := ts.GetSize(); err == nil {
-		termWidth = termSize.Col()
+	if width, err := pb.GetTerminalWidth(); err == nil {
+		termWidth = width
 	}
 
 	for {

posix-list-dir-others.go

@@ -21,12 +21,13 @@ package main
 import (
 	"io"
 	"os"
+	"path"
 	"strings"
 )
 
 // Return all the entries at the directory dirPath.
 func readDir(dirPath string) (entries []string, err error) {
-	d, err := os.Open(dirPath)
+	d, err := os.Open(preparePath(dirPath))
 	if err != nil {
 		// File is really not found.
 		if os.IsNotExist(err) {
@@ -42,6 +43,7 @@ func readDir(dirPath string) (entries []string, err error) {
 	defer d.Close()
 
 	for {
+		// Read 1000 entries.
 		fis, err := d.Readdir(1000)
 		if err != nil {
 			if err == io.EOF {
@@ -50,17 +52,33 @@ func readDir(dirPath string) (entries []string, err error) {
 			return nil, err
 		}
 		for _, fi := range fis {
-			// Skip special files.
+			// Skip special files, if found.
 			if hasPosixReservedPrefix(fi.Name()) {
 				continue
 			}
+			// Stat symbolic link and follow to get the final value.
+			if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+				var st os.FileInfo
+				st, err = os.Stat(preparePath(path.Join(dirPath, fi.Name())))
+				if err != nil {
+					errorIf(err, "Unable to stat path %s", path.Join(dirPath, fi.Name()))
+					continue
+				}
+				// Append to entries if symbolic link exists and is valid.
+				if st.IsDir() {
+					entries = append(entries, fi.Name()+slashSeparator)
+				} else if st.Mode().IsRegular() {
+					entries = append(entries, fi.Name())
+				}
+				continue
+			}
 			if fi.Mode().IsDir() {
-				// append "/" instead of "\" so that sorting is done as expected.
+				// Append "/" instead of "\" so that sorting is achieved as expected.
 				entries = append(entries, fi.Name()+slashSeparator)
 			} else if fi.Mode().IsRegular() {
 				entries = append(entries, fi.Name())
 			}
 		}
 	}
-	return
+	return entries, nil
 }

posix-list-dir_test.go

@@ -0,0 +1,225 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"testing"
+)
+
+// Test to check for different input arguments.
+func TestReadDirFail(t *testing.T) {
+	// Check non existent directory.
+	if _, err := readDir("/tmp/non-existent-directory"); err != errFileNotFound {
+		t.Fatalf("expected = %s, got: %s", errFileNotFound, err)
+	}
+
+	// Check if file is given.
+	if _, err := readDir("/etc/issue/mydir"); err != errFileNotFound {
+		t.Fatalf("expected = %s, got: %s", errFileNotFound, err)
+	}
+
+	// Only valid for linux.
+	if runtime.GOOS == "linux" {
+		// Check if permission denied.
+		if _, err := readDir("/proc/1/fd"); err == nil {
+			t.Fatalf("expected = an error, got: nil")
+		}
+	}
+}
+
+// Represents data type for all the test results.
+type result struct {
+	dir     string
+	entries []string
+}
+
+func mustSetupDir(t *testing.T) string {
+	// Create unique test directory.
+	dir, err := ioutil.TempDir("", "minio-posix-list-dir")
+	if err != nil {
+		t.Fatalf("Unable to setup directory, %s", err)
+	}
+	return dir
+}
+
+// Test to read empty directory.
+func setupTestReadDirEmpty(t *testing.T) (testResults []result) {
+	// Add empty entry slice for this test directory.
+	testResults = append(testResults, result{mustSetupDir(t), []string{}})
+	return testResults
+}
+
+// Test to read empty directory with only reserved names.
+func setupTestReadDirReserved(t *testing.T) (testResults []result) {
+	dir := mustSetupDir(t)
+	entries := []string{}
+	// Create a file with reserved name.
+	for _, reservedName := range posixReservedPrefix {
+		if err := ioutil.WriteFile(filepath.Join(dir, reservedName), []byte{}, os.ModePerm); err != nil {
+			// For cleanup, its required to add these entries into test results.
+			testResults = append(testResults, result{dir, entries})
+			t.Fatalf("Unable to create file, %s", err)
+		}
+		// entries = append(entries, reservedName) - reserved files are skipped.
+	}
+	sort.Strings(entries)
+
+	// Add entries slice for this test directory.
+	testResults = append(testResults, result{dir, entries})
+	return testResults
+}
+
+// Test to read non-empty directory with only files.
+func setupTestReadDirFiles(t *testing.T) (testResults []result) {
+	dir := mustSetupDir(t)
+	entries := []string{}
+	for i := 0; i < 10; i++ {
+		name := fmt.Sprintf("file-%d", i)
+		if err := ioutil.WriteFile(filepath.Join(dir, name), []byte{}, os.ModePerm); err != nil {
+			// For cleanup, its required to add these entries into test results.
+			testResults = append(testResults, result{dir, entries})
+			t.Fatalf("Unable to create file, %s", err)
+		}
+		entries = append(entries, name)
+	}
+
+	// Keep entries sorted for easier comparison.
+	sort.Strings(entries)
+
+	// Add entries slice for this test directory.
+	testResults = append(testResults, result{dir, entries})
+	return testResults
+}
+
+// Test to read non-empty directory with directories and files.
+func setupTestReadDirGeneric(t *testing.T) (testResults []result) {
+	dir := mustSetupDir(t)
+	if err := os.MkdirAll(filepath.Join(dir, "mydir"), 0777); err != nil {
+		t.Fatalf("Unable to create prefix directory \"mydir\", %s", err)
+	}
+	entries := []string{"mydir/"}
+	for i := 0; i < 10; i++ {
+		name := fmt.Sprintf("file-%d", i)
+		if err := ioutil.WriteFile(filepath.Join(dir, "mydir", name), []byte{}, os.ModePerm); err != nil {
+			// For cleanup, its required to add these entries into test results.
+			testResults = append(testResults, result{dir, entries})
+			t.Fatalf("Unable to write file, %s", err)
+		}
+	}
+	// Keep entries sorted for easier comparison.
+	sort.Strings(entries)
+
+	// Add entries slice for this test directory.
+	testResults = append(testResults, result{dir, entries})
+	return testResults
+}
+
+// Test to read non-empty directory with symlinks.
+func setupTestReadDirSymlink(t *testing.T) (testResults []result) {
+	dir := mustSetupDir(t)
+	entries := []string{}
+	for i := 0; i < 10; i++ {
+		name1 := fmt.Sprintf("file-%d", i)
+		name2 := fmt.Sprintf("file-%d", i+10)
+		if err := ioutil.WriteFile(filepath.Join(dir, name1), []byte{}, os.ModePerm); err != nil {
+			// For cleanup, its required to add these entries into test results.
+			testResults = append(testResults, result{dir, entries})
+			t.Fatalf("Unable to create a file, %s", err)
+		}
+		// Symlink will not be added to entries.
+		if err := os.Symlink(filepath.Join(dir, name1), filepath.Join(dir, name2)); err != nil {
+			t.Fatalf("Unable to create a symlink, %s", err)
+		}
+		// Add to entries.
+		entries = append(entries, name1)
+		entries = append(entries, name2)
+	}
+	if err := os.MkdirAll(filepath.Join(dir, "mydir"), 0777); err != nil {
+		t.Fatalf("Unable to create \"mydir\", %s", err)
+	}
+	entries = append(entries, "mydir/")
+
+	// Keep entries sorted for easier comparison.
+	sort.Strings(entries)
+
+	// Add entries slice for this test directory.
+	testResults = append(testResults, result{dir, entries})
+	return testResults
+}
+
+// checkResult - checks whether entries are got are same as expected entries.
+func checkResult(expected []string, got []string) bool {
+	// If length of expected and got slice are different, the test actually failed.
+	if len(expected) != len(got) {
+		return false
+	}
+
+	for i := range expected {
+		// If entry in expected is not same as entry it got, the test is failed.
+		if expected[i] != got[i] {
+			return false
+		}
+	}
+
+	// expected and got have same entries.
+	return true
+}
+
+// teardown - cleans up test directories.
+func teardown(testResults []result) {
+	for _, r := range testResults {
+		os.RemoveAll(r.dir)
+	}
+}
+
+// TestReadDir - test function to run various readDir() tests.
+func TestReadDir(t *testing.T) {
+	var testResults []result
+
+	// Setup and capture test results for empty directory.
+	testResults = append(testResults, setupTestReadDirEmpty(t)...)
+	// Setup and capture test results for reserved files.
+	testResults = append(testResults, setupTestReadDirReserved(t)...)
+	// Setup and capture test results for directory with only files.
+	testResults = append(testResults, setupTestReadDirFiles(t)...)
+	// Setup and capture test results for directory with files and directories.
+	testResults = append(testResults, setupTestReadDirGeneric(t)...)
+	// Setup and capture test results for directory with files and symlink.
+	testResults = append(testResults, setupTestReadDirSymlink(t)...)
+
+	// Remove all dirs once tests are over.
+	defer teardown(testResults)
+
+	// Validate all the results.
+	for _, r := range testResults {
+		if entries, err := readDir(r.dir); err != nil {
+			t.Fatal("failed to run test.", err)
+		} else {
+			// Keep entries sorted for easier comparison.
+			sort.Strings(entries)
+			if !checkResult(r.entries, entries) {
+				t.Fatalf("expected = %s, got: %s", r.entries, entries)
+			}
+		}
+	}
+}

posix-prepare-path.go

@@ -0,0 +1,56 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+// preparePath rewrites path to handle any OS specific details.
+func preparePath(path string) string {
+	if runtime.GOOS == "windows" {
+		// Microsoft Windows supports long path names using
+		// uniform naming convention (UNC).
+		return UNCPath(path)
+	}
+	return path
+}
+
+// UNCPath converts a absolute windows path to a UNC long path.
+func UNCPath(path string) string {
+	// Clean the path for any trailing "/".
+	path = filepath.Clean(path)
+
+	// UNC can NOT use "/", so convert all to "\".
+	path = filepath.FromSlash(path)
+
+	// If prefix is "\\", we already have a UNC path or server.
+	if strings.HasPrefix(path, `\\`) {
+
+		// If already long path, just keep it
+		if strings.HasPrefix(path, `\\?\`) {
+			return path
+		}
+
+		// Trim "\\" from path and add UNC prefix.
+		return `\\?\UNC\` + strings.TrimPrefix(path, `\\`)
+	}
+	path = `\\?\` + path
+	return path
+}

posix-utils_common.go

@@ -16,42 +16,7 @@
 
 package main
 
-import (
-	"runtime"
-	"strings"
-	"unicode/utf8"
-)
-
-// isValidVolname verifies a volname name in accordance with object
-// layer requirements.
-func isValidVolname(volname string) bool {
-	if len(volname) < 3 || len(volname) > 63 {
-		return false
-	}
-	switch runtime.GOOS {
-	case "windows":
-		// Volname shouldn't have reserved characters on windows in it.
-		return !strings.ContainsAny(volname, "/\\:*?\"<>|")
-	default:
-		// Volname shouldn't have '/' in it.
-		return !strings.ContainsAny(volname, "/")
-	}
-}
-
-// Keeping this as lower bound value supporting Linux, Darwin and Windows operating systems.
-const pathMax = 4096
-
-// isValidPath verifies if a path name is in accordance with FS limitations.
-func isValidPath(path string) bool {
-	// TODO: Make this FSType or Operating system specific.
-	if len(path) > pathMax || len(path) == 0 {
-		return false
-	}
-	if !utf8.ValidString(path) {
-		return false
-	}
-	return true
-}
+import "strings"
 
 // List of reserved words for files, includes old and new ones.
 var posixReservedPrefix = []string{

posix-utils_nix.go

@@ -0,0 +1,44 @@
+// +build linux darwin dragonfly freebsd netbsd openbsd
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "os"
+
+// isValidVolname verifies a volname name in accordance with object
+// layer requirements.
+func isValidVolname(volname string) bool {
+	return !(len(volname) < 3 || len(volname) > 63)
+}
+
+// mkdirAll creates a directory named path,
+// along with any necessary parents, and returns nil,
+// or else returns an error. The permission bits perm are used
+// for all directories that mkdirAll creates. If path is already
+// a directory, mkdirAll does nothing and returns nil.
+func mkdirAll(path string, perm os.FileMode) error {
+	return os.MkdirAll(path, perm)
+}
+
+// removeAll removes path and any children it contains.
+// It removes everything it can but returns the first error
+// it encounters.  If the path does not exist, RemoveAll
+// returns nil (no error).
+func removeAll(path string) error {
+	return os.RemoveAll(path)
+}

posix-utils_nix_test.go

@@ -0,0 +1,126 @@
+// +build linux darwin dragonfly freebsd netbsd openbsd
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"syscall"
+	"testing"
+)
+
+// Based on `man getumask` a vaporware GNU extension to glibc.
+// returns file mode creation mask.
+func getUmask() int {
+	mask := syscall.Umask(0)
+	syscall.Umask(mask)
+	return mask
+}
+
+// Tests if the directory and file creations happen with proper umask.
+func TestIsValidUmaskVol(t *testing.T) {
+	tmpPath, err := ioutil.TempDir(os.TempDir(), "minio-")
+	if err != nil {
+		t.Fatalf("Initializing temporary directory failed with %s.", err)
+	}
+	testCases := []struct {
+		volName       string
+		expectedUmask int
+	}{
+		{"is-this-valid", getUmask()},
+	}
+	testCase := testCases[0]
+
+	// Initialize a new posix layer.
+	disk, err := newPosix(tmpPath)
+	if err != nil {
+		t.Fatalf("Initializing posix failed with %s.", err)
+	}
+
+	// Attempt to create a volume to verify the permissions later.
+	// MakeVol creates 0777.
+	if err = disk.MakeVol(testCase.volName); err != nil {
+		t.Fatalf("Creating a volume failed with %s expected to pass.", err)
+	}
+	defer removeAll(tmpPath)
+
+	// Stat to get permissions bits.
+	st, err := os.Stat(path.Join(tmpPath, testCase.volName))
+	if err != nil {
+		t.Fatalf("Stat failed with %s expected to pass.", err)
+	}
+
+	// Get umask of the bits stored.
+	currentUmask := 0777 - uint32(st.Mode().Perm())
+
+	// Verify if umask is correct.
+	if int(currentUmask) != testCase.expectedUmask {
+		t.Fatalf("Umask check failed expected %d, got %d", testCase.expectedUmask, currentUmask)
+	}
+}
+
+// Tests if the file creations happen with proper umask.
+func TestIsValidUmaskFile(t *testing.T) {
+	tmpPath, err := ioutil.TempDir(os.TempDir(), "minio-")
+	if err != nil {
+		t.Fatalf("Initializing temporary directory failed with %s.", err)
+	}
+	testCases := []struct {
+		volName       string
+		expectedUmask int
+	}{
+		{"is-this-valid", getUmask()},
+	}
+	testCase := testCases[0]
+
+	// Initialize a new posix layer.
+	disk, err := newPosix(tmpPath)
+	if err != nil {
+		t.Fatalf("Initializing posix failed with %s.", err)
+	}
+
+	// Attempt to create a volume to verify the permissions later.
+	// MakeVol creates directory with 0777 perms.
+	if err = disk.MakeVol(testCase.volName); err != nil {
+		t.Fatalf("Creating a volume failed with %s expected to pass.", err)
+	}
+
+	defer removeAll(tmpPath)
+
+	// Attempt to create a file to verify the permissions later.
+	// AppendFile creates file with 0666 perms.
+	if err = disk.AppendFile(testCase.volName, "hello-world.txt", []byte("Hello World")); err != nil {
+		t.Fatalf("Create a file `test` failed with %s expected to pass.", err)
+	}
+
+	// StatFile - stat the file.
+	fi, err := disk.StatFile(testCase.volName, "hello-world.txt")
+	if err != nil {
+		t.Fatalf("Stat failed with %s expected to pass.", err)
+	}
+
+	// Get umask of the bits stored.
+	currentUmask := 0666 - uint32(fi.Mode.Perm())
+
+	// Verify if umask is correct.
+	if int(currentUmask) != testCase.expectedUmask {
+		t.Fatalf("Umask check failed expected %d, got %d", testCase.expectedUmask, currentUmask)
+	}
+}

posix-utils_test.go

@@ -0,0 +1,73 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "testing"
+
+// Tests validate volume name.
+func TestIsValidVolname(t *testing.T) {
+	testCases := []struct {
+		volName    string
+		shouldPass bool
+	}{
+		// Cases which should pass the test.
+		// passing in valid bucket names.
+		{"lol", true},
+		{"1-this-is-valid", true},
+		{"1-this-too-is-valid-1", true},
+		{"this.works.too.1", true},
+		{"1234567", true},
+		{"123", true},
+		{"s3-eu-west-1.amazonaws.com", true},
+		{"ideas-are-more-powerful-than-guns", true},
+		{"testbucket", true},
+		{"1bucket", true},
+		{"bucket1", true},
+		{"$this-is-not-valid-too", true},
+		{"contains-$-dollar", true},
+		{"contains-^-carrot", true},
+		{"contains-$-dollar", true},
+		{"contains-$-dollar", true},
+		{".starts-with-a-dot", true},
+		{"ends-with-a-dot.", true},
+		{"ends-with-a-dash-", true},
+		{"-starts-with-a-dash", true},
+		{"THIS-BEINGS-WITH-UPPERCASe", true},
+		{"tHIS-ENDS-WITH-UPPERCASE", true},
+		{"ThisBeginsAndEndsWithUpperCase", true},
+		{"una ñina", true},
+		// cases for which test should fail.
+		// passing invalid bucket names.
+		{"", false},
+		{"/", false},
+		{"a", false},
+		{"ab", false},
+		{"ab/", true},
+		{"......", true},
+		{"lalalallalallalalalallalallalala-theString-size-is-greater-than-64", false},
+	}
+
+	for i, testCase := range testCases {
+		isValidVolname := isValidVolname(testCase.volName)
+		if testCase.shouldPass && !isValidVolname {
+			t.Errorf("Test case %d: Expected \"%s\" to be a valid bucket name", i+1, testCase.volName)
+		}
+		if !testCase.shouldPass && isValidVolname {
+			t.Errorf("Test case %d: Expected bucket name \"%s\" to be invalid", i+1, testCase.volName)
+		}
+	}
+}

posix-utils_windows.go

@@ -0,0 +1,165 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+)
+
+// isValidVolname verifies a volname name in accordance with object
+// layer requirements.
+func isValidVolname(volname string) bool {
+	if len(volname) < 3 || len(volname) > 63 {
+		return false
+	}
+	// Volname shouldn't have reserved characters on windows in it.
+	return !strings.ContainsAny(volname, `\:*?\"<>|`)
+}
+
+// mkdirAll creates a directory named path,
+// along with any necessary parents, and returns nil,
+// or else returns an error. The permission bits perm are used
+// for all directories that mkdirAll creates. If path is already
+// a directory, mkdirAll does nothing and returns nil.
+func mkdirAll(path string, perm os.FileMode) error {
+	path = preparePath(path)
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	dir, err := os.Stat(path)
+	if err == nil {
+		if dir.IsDir() {
+			return nil
+		}
+		return &os.PathError{
+			Op:   "mkdir",
+			Path: path,
+			Err:  syscall.ENOTDIR,
+		}
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(path)
+	for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent
+		parent := path[0 : j-1]
+		if parent != filepath.VolumeName(parent) {
+			err = mkdirAll(parent, perm)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	err = os.Mkdir(path, perm)
+	if err != nil {
+		// Handle arguments like "foo/." by
+		// double-checking that directory doesn't exist.
+		dir, err1 := os.Lstat(path)
+		if err1 == nil && dir.IsDir() {
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
+// removeAll removes path and any children it contains.
+// It removes everything it can but returns the first error
+// it encounters.  If the path does not exist, RemoveAll
+// returns nil (no error).
+func removeAll(path string) error {
+	path = preparePath(path)
+	// Simple case: if Remove works, we're done.
+	err := os.Remove(path)
+	if err == nil || os.IsNotExist(err) {
+		return nil
+	}
+
+	// Otherwise, is this a directory we need to recurse into?
+	dir, serr := os.Lstat(path)
+	if serr != nil {
+		if serr, ok := serr.(*os.PathError); ok && (os.IsNotExist(serr.Err) || serr.Err == syscall.ENOTDIR) {
+			return nil
+		}
+		return serr
+	}
+	if !dir.IsDir() {
+		// Not a directory; return the error from Remove.
+		return err
+	}
+
+	// Directory.
+	fd, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Race. It was deleted between the Lstat and Open.
+			// Return nil per RemoveAll's docs.
+			return nil
+		}
+		return err
+	}
+
+	// Remove contents & return first error.
+	err = nil
+	for {
+		names, err1 := fd.Readdirnames(4096) // Get 4k entries.
+		for _, name := range names {
+			err1 = removeAll(path + string(os.PathSeparator) + name)
+			if err == nil {
+				err = err1
+			}
+		}
+		if err1 == io.EOF {
+			break
+		}
+		// If Readdirnames returned an error, use it.
+		if err == nil {
+			err = err1
+		}
+		if len(names) == 0 {
+			break
+		}
+	}
+
+	// Close directory, because windows won't remove opened directory.
+	fd.Close()
+
+	// Remove directory.
+	err1 := os.Remove(path)
+	if err1 == nil || os.IsNotExist(err1) {
+		return nil
+	}
+	if err == nil {
+		err = err1
+	}
+	return err
+}

posix-utils_windows_test.go

@@ -0,0 +1,160 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+)
+
+// Test if various paths work as expected when converted to UNC form
+func TestUNCPaths(t *testing.T) {
+	var testCases = []struct {
+		objName string
+		pass    bool
+	}{
+		{"/abcdef", true},
+		{"/a/b/c/d/e/f/g", true},
+		{string(bytes.Repeat([]byte("界"), 85)), true},
+		// Each path component must be <= 255 bytes long.
+		{string(bytes.Repeat([]byte("界"), 100)), false},
+		{`\\p\q\r\s\t`, true},
+	}
+	// Instantiate posix object to manage a disk
+	var err error
+	err = os.Mkdir("c:\\testdisk", 0700)
+
+	// Cleanup on exit of test
+	defer os.RemoveAll("c:\\testdisk")
+
+	var fs StorageAPI
+	fs, err = newPosix("c:\\testdisk")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create volume to use in conjunction with other StorageAPI's file API(s)
+	err = fs.MakeVol("voldir")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range testCases {
+		err = fs.AppendFile("voldir", test.objName, []byte("hello"))
+		if err != nil && test.pass {
+			t.Error(err)
+		} else if err == nil && !test.pass {
+			t.Error(err)
+		}
+
+		fs.DeleteFile("voldir", test.objName)
+	}
+}
+
+// Test to validate posix behaviour on windows when a non-final path component is a file.
+func TestUNCPathENOTDIR(t *testing.T) {
+	var err error
+	// Instantiate posix object to manage a disk
+	err = os.Mkdir("c:\\testdisk", 0700)
+
+	// Cleanup on exit of test
+	defer os.RemoveAll("c:\\testdisk")
+	var fs StorageAPI
+	fs, err = newPosix("c:\\testdisk")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create volume to use in conjunction with other StorageAPI's file API(s)
+	err = fs.MakeVol("voldir")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = fs.AppendFile("voldir", "/file", []byte("hello"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Try to create a file that includes a file in its path components.
+	// In *nix, this returns syscall.ENOTDIR while in windows we receive the following error.
+	err = fs.AppendFile("voldir", "/file/obj1", []byte("hello"))
+	if err != errFileAccessDenied {
+		t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
+	}
+}
+
+// Test to validate that path name in UNC form works
+func TestUNCPathDiskName(t *testing.T) {
+	var err error
+	// Instantiate posix object to manage a disk
+	longPathDisk := `\\?\c:\testdisk`
+	err = mkdirAll(longPathDisk, 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Cleanup on exit of test
+	defer removeAll(longPathDisk)
+	var fs StorageAPI
+	fs, err = newPosix(longPathDisk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create volume to use in conjunction with other StorageAPI's file API(s)
+	err = fs.MakeVol("voldir")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Test to validate 32k path works on windows platform
+func Test32kUNCPath(t *testing.T) {
+	var err error
+	// Instantiate posix object to manage a disk
+	longDiskName := `\\?\c:`
+	for {
+		compt := strings.Repeat("a", 255)
+		if len(compt)+len(longDiskName)+1 > 32767 {
+			break
+		}
+		longDiskName = longDiskName + `\` + compt
+	}
+
+	if len(longDiskName) < 32767 {
+		// The following calculation was derived empirically. It is not exactly MAX_PATH - len(longDiskName)
+		// possibly due to expansion rules as mentioned here -
+		// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
+		remaining := 32767 - 25 - len(longDiskName) - 10
+		longDiskName = longDiskName + `\` + strings.Repeat("a", remaining)
+	}
+	err = mkdirAll(longDiskName, 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Cleanup on exit of test
+	defer removeAll(longDiskName)
+	_, err = newPosix(longDiskName)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

posix.go

@@ -18,12 +18,15 @@ package main
 
 import (
 	"bytes"
+	"errors"
 	"io"
+	"io/ioutil"
 	"os"
 	slashpath "path"
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync/atomic"
 	"syscall"
 
 	"github.com/minio/minio/pkg/disk"
@@ -31,26 +34,26 @@ import (
 
 const (
 	fsMinSpacePercent = 5
+	maxAllowedIOError = 5
 )
 
 // posix - implements StorageAPI interface.
 type posix struct {
+	ioErrCount  int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 	diskPath    string
 	minFreeDisk int64
 }
 
+var errFaultyDisk = errors.New("Faulty disk")
+
 // checkPathLength - returns error if given path name length more than 255
 func checkPathLength(pathName string) error {
-	// For MS Windows, the maximum path length is 255
-	if runtime.GOOS == "windows" {
-		if len(pathName) > 255 {
-			return errFileNameTooLong
-		}
-
-		return nil
+	// Apple OS X path length is limited to 1016
+	if runtime.GOOS == "darwin" && len(pathName) > 1016 {
+		return errFileNameTooLong
 	}
 
-	// For non-windows system, check each path segment length is > 255
+	// Check each path segment length is > 255
 	for len(pathName) > 0 && pathName != "." && pathName != "/" {
 		dir, file := slashpath.Dir(pathName), slashpath.Base(pathName)
 
@@ -59,8 +62,7 @@ func checkPathLength(pathName string) error {
 		}
 
 		pathName = dir
-	}
-
+	} // Success.
 	return nil
 }
 
@@ -91,14 +93,24 @@ func newPosix(diskPath string) (StorageAPI, error) {
 	if diskPath == "" {
 		return nil, errInvalidArgument
 	}
-	fs := posix{
+	var err error
+	// Disallow relative paths, figure out absolute paths.
+	diskPath, err = filepath.Abs(diskPath)
+	if err != nil {
+		return nil, err
+	}
+	fs := &posix{
 		diskPath:    diskPath,
 		minFreeDisk: fsMinSpacePercent, // Minimum 5% disk should be free.
 	}
-	st, err := os.Stat(diskPath)
+	st, err := os.Stat(preparePath(diskPath))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return fs, errDiskNotFound
+			// Disk not found create it.
+			if err = os.MkdirAll(diskPath, 0777); err != nil {
+				return fs, err
+			}
+			return fs, nil
 		}
 		return fs, err
 	}
@@ -108,23 +120,31 @@ func newPosix(diskPath string) (StorageAPI, error) {
 	return fs, nil
 }
 
-// checkDiskFree verifies if disk path has sufficient minium free disk space.
-func checkDiskFree(diskPath string, minFreeDisk int64) (err error) {
-	if err = checkPathLength(diskPath); err != nil {
-		return err
+// getDiskInfo returns given disk information.
+func getDiskInfo(diskPath string) (di disk.Info, err error) {
+	if err = checkPathLength(diskPath); err == nil {
+		di, err = disk.GetInfo(diskPath)
 	}
-	di, err := disk.GetInfo(diskPath)
+
+	if os.IsNotExist(err) {
+		err = errDiskNotFound
+	}
+
+	return di, err
+}
+
+// checkDiskFree verifies if disk path has sufficient minimum free disk space and files.
+func checkDiskFree(diskPath string, minFreeDisk int64) (err error) {
+	di, err := getDiskInfo(diskPath)
 	if err != nil {
-		if os.IsNotExist(err) {
-			return errDiskNotFound
-		}
 		return err
 	}
 
 	// Remove 5% from total space for cumulative disk
 	// space used for journalling, inodes etc.
 	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
-	if int64(availableDiskSpace) <= minFreeDisk {
+	availableFiles := (float64(di.Ffree) / (float64(di.Files) - (0.05 * float64(di.Files)))) * 100
+	if int64(availableDiskSpace) <= minFreeDisk || int64(availableFiles) <= minFreeDisk {
 		return errDiskFull
 	}
 
@@ -148,7 +168,7 @@ func listVols(dirPath string) ([]VolInfo, error) {
 			continue
 		}
 		var fi os.FileInfo
-		fi, err = os.Stat(pathJoin(dirPath, entry))
+		fi, err = os.Stat(preparePath(pathJoin(dirPath, entry)))
 		if err != nil {
 			// If the file does not exist, skip the entry.
 			if os.IsNotExist(err) {
@@ -170,7 +190,7 @@ func listVols(dirPath string) ([]VolInfo, error) {
 // corresponding valid volume names on the backend in a platform
 // compatible way for all operating systems. If volume is not found
 // an error is generated.
-func (s posix) getVolDir(volume string) (string, error) {
+func (s *posix) getVolDir(volume string) (string, error) {
 	if !isValidVolname(volume) {
 		return "", errInvalidArgument
 	}
@@ -182,7 +202,17 @@ func (s posix) getVolDir(volume string) (string, error) {
 }
 
 // Make a volume entry.
-func (s posix) MakeVol(volume string) (err error) {
+func (s *posix) MakeVol(volume string) (err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return errFaultyDisk
+	}
+
 	// Validate if disk is free.
 	if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
 		return err
@@ -192,17 +222,32 @@ func (s posix) MakeVol(volume string) (err error) {
 	if err != nil {
 		return err
 	}
-	// Make a volume entry.
-	err = os.Mkdir(volumeDir, 0700)
-	if err != nil && os.IsExist(err) {
-		return errVolumeExists
+	// Make a volume entry, with mode 0777 mkdir honors system umask.
+	err = os.Mkdir(preparePath(volumeDir), 0777)
+	if err != nil {
+		if os.IsExist(err) {
+			return errVolumeExists
+		} else if os.IsPermission(err) {
+			return errDiskAccessDenied
+		}
+		return err
 	}
 	// Success
 	return nil
 }
 
 // ListVols - list volumes.
-func (s posix) ListVols() (volsInfo []VolInfo, err error) {
+func (s *posix) ListVols() (volsInfo []VolInfo, err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return nil, errFaultyDisk
+	}
+
 	volsInfo, err = listVols(s.diskPath)
 	if err != nil {
 		return nil, err
@@ -218,9 +263,19 @@ func (s posix) ListVols() (volsInfo []VolInfo, err error) {
 }
 
 // StatVol - get volume info.
-func (s posix) StatVol(volume string) (volInfo VolInfo, err error) {
-	// Validate if disk is free.
-	if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
+func (s *posix) StatVol(volume string) (volInfo VolInfo, err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return VolInfo{}, errFaultyDisk
+	}
+
+	// Check disk availability.
+	if _, err = getDiskInfo(s.diskPath); err != nil {
 		return VolInfo{}, err
 	}
 
@@ -231,7 +286,7 @@ func (s posix) StatVol(volume string) (volInfo VolInfo, err error) {
 	}
 	// Stat a volume entry.
 	var st os.FileInfo
-	st, err = os.Stat(volumeDir)
+	st, err = os.Stat(preparePath(volumeDir))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return VolInfo{}, errVolumeNotFound
@@ -248,9 +303,19 @@ func (s posix) StatVol(volume string) (volInfo VolInfo, err error) {
 }
 
 // DeleteVol - delete a volume.
-func (s posix) DeleteVol(volume string) error {
-	// Validate if disk is free.
-	if err := checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
+func (s *posix) DeleteVol(volume string) (err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return errFaultyDisk
+	}
+
+	// Check disk availability.
+	if _, err = getDiskInfo(s.diskPath); err != nil {
 		return err
 	}
 
@@ -259,7 +324,7 @@ func (s posix) DeleteVol(volume string) error {
 	if err != nil {
 		return err
 	}
-	err = os.Remove(volumeDir)
+	err = os.Remove(preparePath(volumeDir))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
@@ -278,9 +343,19 @@ func (s posix) DeleteVol(volume string) error {
 
 // ListDir - return all the entries at the given directory path.
 // If an entry is a directory it will be returned with a trailing "/".
-func (s posix) ListDir(volume, dirPath string) ([]string, error) {
-	// Validate if disk is free.
-	if err := checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
+func (s *posix) ListDir(volume, dirPath string) (entries []string, err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return nil, errFaultyDisk
+	}
+
+	// Check disk availability.
+	if _, err = getDiskInfo(s.diskPath); err != nil {
 		return nil, err
 	}
 
@@ -290,7 +365,7 @@ func (s posix) ListDir(volume, dirPath string) ([]string, error) {
 		return nil, err
 	}
 	// Stat a volume entry.
-	_, err = os.Stat(volumeDir)
+	_, err = os.Stat(preparePath(volumeDir))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, errVolumeNotFound
@@ -300,14 +375,89 @@ func (s posix) ListDir(volume, dirPath string) ([]string, error) {
 	return readDir(pathJoin(volumeDir, dirPath))
 }
 
+// ReadAll reads from r until an error or EOF and returns the data it read.
+// A successful call returns err == nil, not err == EOF. Because ReadAll is
+// defined to read from src until EOF, it does not treat an EOF from Read
+// as an error to be reported.
+// This API is meant to be used on files which have small memory footprint, do
+// not use this on large files as it would cause server to crash.
+func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return nil, errFaultyDisk
+	}
+
+	// Check disk availability.
+	if _, err = getDiskInfo(s.diskPath); err != nil {
+		return nil, err
+	}
+
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return nil, err
+	}
+	// Stat a volume entry.
+	_, err = os.Stat(preparePath(volumeDir))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, errVolumeNotFound
+		}
+		return nil, err
+	}
+
+	// Validate file path length, before reading.
+	filePath := pathJoin(volumeDir, path)
+	if err = checkPathLength(filePath); err != nil {
+		return nil, err
+	}
+
+	// Open the file for reading.
+	buf, err = ioutil.ReadFile(preparePath(filePath))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, errFileNotFound
+		} else if os.IsPermission(err) {
+			return nil, errFileAccessDenied
+		} else if pathErr, ok := err.(*os.PathError); ok {
+			switch pathErr.Err {
+			case syscall.ENOTDIR, syscall.EISDIR:
+				return nil, errFileNotFound
+			default:
+				if strings.Contains(pathErr.Err.Error(), "The handle is invalid") {
+					// This case is special and needs to be handled for windows.
+					return nil, errFileNotFound
+				}
+			}
+			return nil, pathErr
+		}
+		return nil, err
+	}
+	return buf, nil
+}
+
 // ReadFile reads exactly len(buf) bytes into buf. It returns the
 // number of bytes copied. The error is EOF only if no bytes were
 // read. On return, n == len(buf) if and only if err == nil. n == 0
 // for io.EOF. Additionally ReadFile also starts reading from an
 // offset.
-func (s posix) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
-	// Validate if disk is free.
-	if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
+func (s *posix) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return 0, errFaultyDisk
+	}
+
+	// Check disk availability.
+	if _, err = getDiskInfo(s.diskPath); err != nil {
 		return 0, err
 	}
 
@@ -316,7 +466,7 @@ func (s posix) ReadFile(volume string, path string, offset int64, buf []byte) (n
 		return 0, err
 	}
 	// Stat a volume entry.
-	_, err = os.Stat(volumeDir)
+	_, err = os.Stat(preparePath(volumeDir))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return 0, errVolumeNotFound
@@ -324,11 +474,14 @@ func (s posix) ReadFile(volume string, path string, offset int64, buf []byte) (n
 		return 0, err
 	}
 
+	// Validate effective path length before reading.
 	filePath := pathJoin(volumeDir, path)
 	if err = checkPathLength(filePath); err != nil {
 		return 0, err
 	}
-	file, err := os.Open(filePath)
+
+	// Open the file for reading.
+	file, err := os.Open(preparePath(filePath))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return 0, errFileNotFound
@@ -356,75 +509,99 @@ func (s posix) ReadFile(volume string, path string, offset int64, buf []byte) (n
 	// Close the reader.
 	defer file.Close()
 
-	// Read file.
+	// Read full until buffer.
 	m, err := io.ReadFull(file, buf)
 
-	// Error unexpected is valid, set this back to nil.
-	if err == io.ErrUnexpectedEOF {
-		err = nil
-	}
-
 	// Success.
 	return int64(m), err
 }
 
 // AppendFile - append a byte array at path, if file doesn't exist at
 // path this call explicitly creates it.
-func (s posix) AppendFile(volume, path string, buf []byte) (n int64, err error) {
+func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return errFaultyDisk
+	}
+
 	// Validate if disk is free.
 	if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
-		return 0, err
+		return err
 	}
 
 	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
-		return 0, err
+		return err
 	}
 	// Stat a volume entry.
-	_, err = os.Stat(volumeDir)
+	_, err = os.Stat(preparePath(volumeDir))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return 0, errVolumeNotFound
+			return errVolumeNotFound
 		}
-		return 0, err
-	}
-	if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
-		return 0, err
+		return err
 	}
 	filePath := pathJoin(volumeDir, path)
 	if err = checkPathLength(filePath); err != nil {
-		return 0, err
+		return err
 	}
 	// Verify if the file already exists and is not of regular type.
 	var st os.FileInfo
-	if st, err = os.Stat(filePath); err == nil {
+	if st, err = os.Stat(preparePath(filePath)); err == nil {
 		if st.IsDir() {
-			return 0, errIsNotRegular
+			return errIsNotRegular
 		}
 	}
 	// Create top level directories if they don't exist.
-	if err = os.MkdirAll(filepath.Dir(filePath), 0700); err != nil {
-		return 0, err
+	// with mode 0777 mkdir honors system umask.
+	if err = mkdirAll(filepath.Dir(filePath), 0777); err != nil {
+		// File path cannot be verified since one of the parents is a file.
+		if strings.Contains(err.Error(), "not a directory") {
+			return errFileAccessDenied
+		} else if runtime.GOOS == "windows" && strings.Contains(err.Error(), "system cannot find the path specified") {
+			return errFileAccessDenied
+		}
+		return err
 	}
-	w, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+
+	// Creates the named file with mode 0666 (before umask), or starts appending
+	// to an existig file.
+	w, err := os.OpenFile(preparePath(filePath), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
 	if err != nil {
 		// File path cannot be verified since one of the parents is a file.
 		if strings.Contains(err.Error(), "not a directory") {
-			return 0, errFileAccessDenied
+			return errFileAccessDenied
 		}
-		return 0, err
+		return err
 	}
+
 	// Close upon return.
 	defer w.Close()
 
 	// Return io.Copy
-	return io.Copy(w, bytes.NewReader(buf))
+	_, err = io.Copy(w, bytes.NewReader(buf))
+	return err
 }
 
 // StatFile - get file info.
-func (s posix) StatFile(volume, path string) (file FileInfo, err error) {
-	// Validate if disk is free.
-	if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
+func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return FileInfo{}, errFaultyDisk
+	}
+
+	// Check disk availability.
+	if _, err = getDiskInfo(s.diskPath); err != nil {
 		return FileInfo{}, err
 	}
 
@@ -433,7 +610,7 @@ func (s posix) StatFile(volume, path string) (file FileInfo, err error) {
 		return FileInfo{}, err
 	}
 	// Stat a volume entry.
-	_, err = os.Stat(volumeDir)
+	_, err = os.Stat(preparePath(volumeDir))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return FileInfo{}, errVolumeNotFound
@@ -445,7 +622,7 @@ func (s posix) StatFile(volume, path string) (file FileInfo, err error) {
 	if err = checkPathLength(filePath); err != nil {
 		return FileInfo{}, err
 	}
-	st, err := os.Stat(filePath)
+	st, err := os.Stat(preparePath(filePath))
 	if err != nil {
 		// File is really not found.
 		if os.IsNotExist(err) {
@@ -479,7 +656,7 @@ func deleteFile(basePath, deletePath string) error {
 		return nil
 	}
 	// Verify if the path exists.
-	pathSt, err := os.Stat(deletePath)
+	pathSt, err := os.Stat(preparePath(deletePath))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errFileNotFound
@@ -493,7 +670,7 @@ func deleteFile(basePath, deletePath string) error {
 		return nil
 	}
 	// Attempt to remove path.
-	if err := os.Remove(deletePath); err != nil {
+	if err := os.Remove(preparePath(deletePath)); err != nil {
 		return err
 	}
 	// Recursively go down the next path and delete again.
@@ -504,9 +681,19 @@ func deleteFile(basePath, deletePath string) error {
 }
 
 // DeleteFile - delete a file at path.
-func (s posix) DeleteFile(volume, path string) error {
-	// Validate if disk is free.
-	if err := checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
+func (s *posix) DeleteFile(volume, path string) (err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return errFaultyDisk
+	}
+
+	// Check disk availability.
+	if _, err = getDiskInfo(s.diskPath); err != nil {
 		return err
 	}
 
@@ -515,7 +702,7 @@ func (s posix) DeleteFile(volume, path string) error {
 		return err
 	}
 	// Stat a volume entry.
-	_, err = os.Stat(volumeDir)
+	_, err = os.Stat(preparePath(volumeDir))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
@@ -535,9 +722,19 @@ func (s posix) DeleteFile(volume, path string) error {
 }
 
 // RenameFile - rename source path to destination path atomically.
-func (s posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
+func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
+	defer func() {
+		if err == syscall.EIO {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if s.ioErrCount > maxAllowedIOError {
+		return errFaultyDisk
+	}
+
 	// Validate if disk is free.
-	if err := checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
+	if err = checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
 		return err
 	}
 
@@ -550,14 +747,14 @@ func (s posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
 		return err
 	}
 	// Stat a volume entry.
-	_, err = os.Stat(srcVolumeDir)
+	_, err = os.Stat(preparePath(srcVolumeDir))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
 		}
 		return err
 	}
-	_, err = os.Stat(dstVolumeDir)
+	_, err = os.Stat(preparePath(dstVolumeDir))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
@@ -570,9 +767,17 @@ func (s posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
 	if !(srcIsDir && dstIsDir || !srcIsDir && !dstIsDir) {
 		return errFileAccessDenied
 	}
+	srcFilePath := slashpath.Join(srcVolumeDir, srcPath)
+	if err = checkPathLength(srcFilePath); err != nil {
+		return err
+	}
+	dstFilePath := slashpath.Join(dstVolumeDir, dstPath)
+	if err = checkPathLength(dstFilePath); err != nil {
+		return err
+	}
 	if srcIsDir {
 		// If source is a directory we expect the destination to be non-existent always.
-		_, err = os.Stat(slashpath.Join(dstVolumeDir, dstPath))
+		_, err = os.Stat(preparePath(dstFilePath))
 		if err == nil {
 			return errFileAccessDenied
 		}
@@ -581,14 +786,21 @@ func (s posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
 		}
 		// Destination does not exist, hence proceed with the rename.
 	}
-	if err = os.MkdirAll(slashpath.Dir(slashpath.Join(dstVolumeDir, dstPath)), 0755); err != nil {
+	// Creates all the parent directories, with mode 0777 mkdir honors system umask.
+	if err = mkdirAll(preparePath(slashpath.Dir(dstFilePath)), 0777); err != nil {
 		// File path cannot be verified since one of the parents is a file.
 		if strings.Contains(err.Error(), "not a directory") {
 			return errFileAccessDenied
+		} else if strings.Contains(err.Error(), "The system cannot find the path specified.") && runtime.GOOS == "windows" {
+			// This is a special case should be handled only for
+			// windows, because windows API does not return "not a
+			// directory" error message. Handle this specifically here.
+			return errFileAccessDenied
 		}
 		return err
 	}
-	err = os.Rename(slashpath.Join(srcVolumeDir, srcPath), slashpath.Join(dstVolumeDir, dstPath))
+	// Finally attempt a rename.
+	err = os.Rename(preparePath(srcFilePath), preparePath(dstFilePath))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errFileNotFound

posix_test.go

@@ -0,0 +1,586 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	slashpath "path"
+	"runtime"
+	"syscall"
+	"testing"
+)
+
+// Tests posix.getDiskInfo()
+func TestGetDiskInfo(t *testing.T) {
+	path, err := ioutil.TempDir(os.TempDir(), "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	testCases := []struct {
+		diskPath    string
+		expectedErr error
+	}{
+		{path, nil},
+		{"/nonexistent-dir", errDiskNotFound},
+	}
+
+	// Check test cases.
+	for _, testCase := range testCases {
+		if _, err := getDiskInfo(testCase.diskPath); err != testCase.expectedErr {
+			t.Fatalf("expected: %s, got: %s", testCase.expectedErr, err)
+		}
+	}
+}
+
+// Tests the functionality implemented by ReadAll storage API.
+func TestReadAll(t *testing.T) {
+	path, err := ioutil.TempDir(os.TempDir(), "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	// Initialize posix storage layer.
+	posix, err := newPosix(path)
+	if err != nil {
+		t.Fatalf("Unable to initialize posix, %s", err)
+	}
+
+	// Create files for the test cases.
+	if err = posix.MakeVol("exists"); err != nil {
+		t.Fatalf("Unable to create a volume \"exists\", %s", err)
+	}
+	if err = posix.AppendFile("exists", "as-directory/as-file", []byte("Hello, World")); err != nil {
+		t.Fatalf("Unable to create a file \"as-directory/as-file\", %s", err)
+	}
+	if err = posix.AppendFile("exists", "as-file", []byte("Hello, World")); err != nil {
+		t.Fatalf("Unable to create a file \"as-file\", %s", err)
+	}
+	if err = posix.AppendFile("exists", "as-file-parent", []byte("Hello, World")); err != nil {
+		t.Fatalf("Unable to create a file \"as-file-parent\", %s", err)
+	}
+
+	// Testcases to validate different conditions for ReadAll API.
+	testCases := []struct {
+		volume string
+		path   string
+		err    error
+	}{
+		// Validate volume does not exist.
+		{
+			"i-dont-exist",
+			"",
+			errVolumeNotFound,
+		},
+		// Validate bad condition file does not exist.
+		{
+			"exists",
+			"as-file-not-found",
+			errFileNotFound,
+		},
+		// Validate bad condition file exists as prefix/directory and
+		// we are attempting to read it.
+		{
+			"exists",
+			"as-directory",
+			errFileNotFound,
+		},
+		{
+			"exists",
+			"as-file-parent/as-file",
+			errFileNotFound,
+		},
+		// Validate the good condition file exists and we are able to read it.
+		{
+			"exists",
+			"as-file",
+			nil,
+		},
+		// Add more cases here.
+	}
+
+	// Run through all the test cases and validate for ReadAll.
+	for i, testCase := range testCases {
+		_, err = posix.ReadAll(testCase.volume, testCase.path)
+		if err != testCase.err {
+			t.Errorf("Test %d expected err %s, got err %s", i+1, testCase.err, err)
+		}
+	}
+}
+
+// TestNewPosix all the cases handled in posix storage layer initialization.
+func TestNewPosix(t *testing.T) {
+	// Temporary dir name.
+	tmpDirName := os.TempDir() + "/" + "minio-" + nextSuffix()
+	// Temporary file name.
+	tmpFileName := os.TempDir() + "/" + "minio-" + nextSuffix()
+	f, _ := os.Create(tmpFileName)
+	f.Close()
+	defer os.Remove(tmpFileName)
+
+	// List of all tests for posix initialization.
+	testCases := []struct {
+		diskPath string
+		err      error
+	}{
+		// Validates input argument cannot be empty.
+		{
+			"",
+			errInvalidArgument,
+		},
+		// Validates if the directory does not exist and
+		// gets automatically created.
+		{
+			tmpDirName,
+			nil,
+		},
+		// Validates if the disk exists as file and returns error
+		// not a directory.
+		{
+			tmpFileName,
+			syscall.ENOTDIR,
+		},
+	}
+
+	// Validate all test cases.
+	for i, testCase := range testCases {
+		_, err := newPosix(testCase.diskPath)
+		if err != testCase.err {
+			t.Fatalf("Test %d failed wanted: %s, got: %s", i+1, err, testCase.err)
+		}
+	}
+}
+
+// Test posix.MakeVol()
+func TestMakeVol(t *testing.T) {
+	// Create temporary directory.
+	path, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	// Initialize posix storage layer.
+	posix, err := newPosix(path)
+	if err != nil {
+		t.Fatalf("Unable to initialize posix, %s", err)
+	}
+
+	// Setup test environment.
+	// Create a file.
+	if err := ioutil.WriteFile(slashpath.Join(path, "vol-as-file"), []byte{}, os.ModePerm); err != nil {
+		t.Fatalf("Unable to create file, %s", err)
+	}
+	// Create a directory.
+	if err := os.Mkdir(slashpath.Join(path, "existing-vol"), 0777); err != nil {
+		t.Fatalf("Unable to create directory, %s", err)
+	}
+
+	testCases := []struct {
+		volName     string
+		expectedErr error
+	}{
+		{"success-vol", nil},
+		{"vol-as-file", errVolumeExists},
+		{"existing-vol", errVolumeExists},
+	}
+
+	for _, testCase := range testCases {
+		if err := posix.MakeVol(testCase.volName); err != testCase.expectedErr {
+			t.Fatalf("case: %s, expected: %s, got: %s", testCase, testCase.expectedErr, err)
+		}
+	}
+
+	// Test for permission denied.
+	if runtime.GOOS != "windows" {
+		// Initialize posix storage layer for permission denied error.
+		posix, err := newPosix("/usr")
+		if err != nil {
+			t.Fatalf("Unable to initialize posix, %s", err)
+		}
+
+		if err := posix.MakeVol("test-vol"); err != errDiskAccessDenied {
+			t.Fatalf("expected: %s, got: %s", errDiskAccessDenied, err)
+		}
+	}
+}
+
+// Test posix.DeleteVol()
+func TestDeleteVol(t *testing.T) {
+	// Create temporary directory.
+	path, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	// Initialize posix storage layer.
+	posix, err := newPosix(path)
+	if err != nil {
+		t.Fatalf("Unable to initialize posix, %s", err)
+	}
+
+	// Setup test environment.
+	if err := posix.MakeVol("success-vol"); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+
+	// Test failure cases.
+	vol := slashpath.Join(path, "nonempty-vol")
+	if err := os.Mkdir(vol, 0777); err != nil {
+		t.Fatalf("Unable to create directory, %s", err)
+	}
+	if err := ioutil.WriteFile(slashpath.Join(vol, "test-file"), []byte{}, os.ModePerm); err != nil {
+		t.Fatalf("Unable to create file, %s", err)
+	}
+
+	testCases := []struct {
+		volName     string
+		expectedErr error
+	}{
+		{"success-vol", nil},
+		{"nonexistent-vol", errVolumeNotFound},
+		{"nonempty-vol", errVolumeNotEmpty},
+	}
+
+	for _, testCase := range testCases {
+		if err := posix.DeleteVol(testCase.volName); err != testCase.expectedErr {
+			t.Fatalf("case: %s, expected: %s, got: %s", testCase, testCase.expectedErr, err)
+		}
+	}
+
+	// Test for permission denied.
+	if runtime.GOOS != "windows" {
+		// Initialize posix storage layer for permission denied error.
+		posix, err := newPosix("/usr")
+		if err != nil {
+			t.Fatalf("Unable to initialize posix, %s", err)
+		}
+
+		if err := posix.DeleteVol("bin"); !os.IsPermission(err) {
+			t.Fatalf("expected: Permission error, got: %s", err)
+		}
+	}
+}
+
+// Test posix.StatVol()
+func TestStatVol(t *testing.T) {
+	// Create temporary directory.
+	path, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	// Initialize posix storage layer.
+	posix, err := newPosix(path)
+	if err != nil {
+		t.Fatalf("Unable to initialize posix, %s", err)
+	}
+
+	// Setup test environment.
+	if err := posix.MakeVol("success-vol"); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+
+	testCases := []struct {
+		volName     string
+		expectedErr error
+	}{
+		{"success-vol", nil},
+		{"nonexistent-vol", errVolumeNotFound},
+	}
+
+	for _, testCase := range testCases {
+		if _, err := posix.StatVol(testCase.volName); err != testCase.expectedErr {
+			t.Fatalf("case: %s, expected: %s, got: %s", testCase, testCase.expectedErr, err)
+		}
+	}
+}
+
+// Test posix.ListVols()
+func TestListVols(t *testing.T) {
+	// Create temporary directory.
+	path, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	// Initialize posix storage layer.
+	posix, err := newPosix(path)
+	if err != nil {
+		t.Fatalf("Unable to initialize posix, %s", err)
+	}
+
+	// Test empty list vols.
+	if volInfo, err := posix.ListVols(); err != nil {
+		t.Fatalf("expected: <nil>, got: %s", err)
+	} else if len(volInfo) != 0 {
+		t.Fatalf("expected: [], got: %s", volInfo)
+	}
+
+	// Test non-empty list vols.
+	if err := posix.MakeVol("success-vol"); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+	if volInfo, err := posix.ListVols(); err != nil {
+		t.Fatalf("expected: <nil>, got: %s", err)
+	} else if len(volInfo) != 1 {
+		t.Fatalf("expected: 1, got: %d", len(volInfo))
+	} else if volInfo[0].Name != "success-vol" {
+		t.Fatalf("expected: success-vol, got: %s", volInfo[0].Name)
+	}
+}
+
+// Test posix.DeleteFile()
+func TestDeleteFile(t *testing.T) {
+	// Create temporary directory.
+	path, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	// Initialize posix storage layer.
+	posix, err := newPosix(path)
+	if err != nil {
+		t.Fatalf("Unable to initialize posix, %s", err)
+	}
+
+	// Setup test environment.
+	if err := posix.MakeVol("success-vol"); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+	if err := posix.AppendFile("success-vol", "success-file", []byte("Hello, world")); err != nil {
+		t.Fatalf("Unable to create file, %s", err)
+	}
+
+	testCases := []struct {
+		fileName    string
+		expectedErr error
+	}{
+		{"success-file", nil},
+		{"nonexistent-file", errFileNotFound},
+	}
+
+	for _, testCase := range testCases {
+		if err := posix.DeleteFile("success-vol", testCase.fileName); err != testCase.expectedErr {
+			t.Fatalf("case: %s, expected: %s, got: %s", testCase, testCase.expectedErr, err)
+		}
+	}
+
+	// Test for permission denied.
+	if runtime.GOOS != "windows" {
+		// Initialize posix storage layer for permission denied error.
+		posix, err := newPosix("/usr")
+		if err != nil {
+			t.Fatalf("Unable to initialize posix, %s", err)
+		}
+
+		if err := posix.DeleteFile("bin", "yes"); !os.IsPermission(err) {
+			t.Fatalf("expected: Permission error, got: %s", err)
+		}
+	}
+}
+
+// Test posix.AppendFile()
+func TestAppendFile(t *testing.T) {
+	// Create temporary directory.
+	path, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	// Initialize posix storage layer.
+	posix, err := newPosix(path)
+	if err != nil {
+		t.Fatalf("Unable to initialize posix, %s", err)
+	}
+
+	// Setup test environment.
+	if err = posix.MakeVol("success-vol"); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+
+	// Create directory to make errIsNotRegular
+	if err = os.Mkdir(slashpath.Join(path, "success-vol", "object-as-dir"), 0777); err != nil {
+		t.Fatalf("Unable to create directory, %s", err)
+	}
+
+	testCases := []struct {
+		fileName    string
+		expectedErr error
+	}{
+		{"myobject", nil},
+		{"path/to/my/object", nil},
+		// Test to append to previously created file.
+		{"myobject", nil},
+		// Test to use same path of previously created file.
+		{"path/to/my/testobject", nil},
+		// One path segment length is 255 chars long.
+		{"path/to/my/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", nil},
+		{"object-as-dir", errIsNotRegular},
+		// path segment uses previously uploaded object.
+		{"myobject/testobject", errFileAccessDenied},
+		// One path segment length is > 255 chars long.
+		{"path/to/my/object0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", errFileNameTooLong},
+		// path length is > 1024 chars long.
+		{"level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", errFileNameTooLong},
+	}
+
+	// Add path length > 1024 test specially as OS X system does not support 1024 long path.
+	err = errFileNameTooLong
+	if runtime.GOOS != "darwin" {
+		err = nil
+	}
+	// path length is 1024 chars long.
+	testCases = append(testCases, struct {
+		fileName    string
+		expectedErr error
+	}{"level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", err})
+
+	for _, testCase := range testCases {
+		if err := posix.AppendFile("success-vol", testCase.fileName, []byte("hello, world")); err != testCase.expectedErr {
+			t.Fatalf("case: %s, expected: %s, got: %s", testCase, testCase.expectedErr, err)
+		}
+	}
+
+	// Test for permission denied.
+	if runtime.GOOS != "windows" {
+		// Initialize posix storage layer for permission denied error.
+		posix, err := newPosix("/usr")
+		if err != nil {
+			t.Fatalf("Unable to initialize posix, %s", err)
+		}
+
+		if err := posix.AppendFile("bin", "yes", []byte("hello, world")); !os.IsPermission(err) {
+			t.Fatalf("expected: Permission error, got: %s", err)
+		}
+	}
+}
+
+// Test posix.RenameFile()
+func TestRenameFile(t *testing.T) {
+	// Create temporary directory.
+	path, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	// Initialize posix storage layer.
+	posix, err := newPosix(path)
+	if err != nil {
+		t.Fatalf("Unable to initialize posix, %s", err)
+	}
+
+	// Setup test environment.
+	if err := posix.MakeVol("src-vol"); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+
+	if err := posix.MakeVol("dest-vol"); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+
+	if err := posix.AppendFile("src-vol", "file1", []byte("Hello, world")); err != nil {
+		t.Fatalf("Unable to create file, %s", err)
+	}
+
+	if err := posix.AppendFile("src-vol", "file2", []byte("Hello, world")); err != nil {
+		t.Fatalf("Unable to create file, %s", err)
+	}
+
+	if err := posix.AppendFile("src-vol", "path/to/file1", []byte("Hello, world")); err != nil {
+		t.Fatalf("Unable to create file, %s", err)
+	}
+
+	testCases := []struct {
+		srcPath     string
+		destPath    string
+		expectedErr error
+	}{
+		{"file1", "file-one", nil},
+		{"path/", "new-path/", nil},
+		// Test to overwrite destination file.
+		{"file2", "file-one", nil},
+		// Test to check failure of source and destination are not same type.
+		{"path/", "file-one", errFileAccessDenied},
+		// Test to check failure of destination directory exists.
+		{"path/", "new-path/", errFileAccessDenied},
+	}
+
+	for _, testCase := range testCases {
+		if err := posix.RenameFile("src-vol", testCase.srcPath, "dest-vol", testCase.destPath); err != testCase.expectedErr {
+			t.Fatalf("case: %s, expected: %s, got: %s", testCase, testCase.expectedErr, err)
+		}
+	}
+}
+
+// Test posix.StatFile()
+func TestStatFile(t *testing.T) {
+	// Create temporary directory.
+	path, err := ioutil.TempDir("", "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create a temporary directory, %s", err)
+	}
+	defer removeAll(path)
+
+	// Initialize posix storage layer.
+	posix, err := newPosix(path)
+	if err != nil {
+		t.Fatalf("Unable to initialize posix, %s", err)
+	}
+
+	// Setup test environment.
+	if err := posix.MakeVol("success-vol"); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+
+	if err := posix.AppendFile("success-vol", "success-file", []byte("Hello, world")); err != nil {
+		t.Fatalf("Unable to create file, %s", err)
+	}
+
+	if err := posix.AppendFile("success-vol", "path/to/success-file", []byte("Hello, world")); err != nil {
+		t.Fatalf("Unable to create file, %s", err)
+	}
+
+	testCases := []struct {
+		path        string
+		expectedErr error
+	}{
+		{"success-file", nil},
+		{"path/to/success-file", nil},
+		// Test nonexistent file.
+		{"nonexistent-file", errFileNotFound},
+		// Test nonexistent path.
+		{"path/2/success-file", errFileNotFound},
+		// Test a directory.
+		{"path", errFileNotFound},
+	}
+
+	for _, testCase := range testCases {
+		if _, err := posix.StatFile("success-vol", testCase.path); err != testCase.expectedErr {
+			t.Fatalf("case: %s, expected: %s, got: %s", testCase, testCase.expectedErr, err)
+		}
+	}
+}

rate-limit-handler.go

@@ -0,0 +1,63 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"net/http"
+	"sync"
+)
+
+// rateLimit - represents datatype of the functionality implemented to
+// limit the number of concurrent http requests.
+type rateLimit struct {
+	handler     http.Handler
+	rqueue      chan struct{}
+	releaseOnce sync.Once
+}
+
+// acquire and release implement a way to send and receive from the
+// channel this is in-turn used to rate limit incoming connections in
+// ServeHTTP() http.Handler method.
+func (c *rateLimit) acquire() { c.rqueue <- struct{}{} }
+func (c *rateLimit) release() { <-c.rqueue }
+
+// ServeHTTP is an http.Handler ServeHTTP method, implemented to rate
+// limit incoming HTTP requests.
+func (c *rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Acquire the connection if queue is not full, otherwise
+	// code path waits here until the previous case is true.
+	c.acquire()
+
+	// Serves the request.
+	c.handler.ServeHTTP(w, r)
+
+	// Release by draining the channel once.
+	c.releaseOnce.Do(c.release)
+}
+
+// setRateLimitHandler limits the number of concurrent http requests based on MINIO_MAXCONN.
+func setRateLimitHandler(handler http.Handler) http.Handler {
+	if globalMaxConn == 0 {
+		return handler
+	} // else proceed to rate limiting.
+
+	// For max connection limit of > '0' we initialize rate limit handler.
+	return &rateLimit{
+		handler: handler,
+		rqueue:  make(chan struct{}, globalMaxConn),
+	}
+}

routers.go

@@ -23,16 +23,15 @@ import (
 	router "github.com/gorilla/mux"
 )
 
-// newObjectLayer - initialize any object layer depending on the
-// number of export paths.
-func newObjectLayer(exportPaths []string) (ObjectLayer, error) {
-	if len(exportPaths) == 1 {
-		exportPath := exportPaths[0]
+// newObjectLayer - initialize any object layer depending on the number of disks.
+func newObjectLayer(disks, ignoredDisks []string) (ObjectLayer, error) {
+	if len(disks) == 1 {
+		exportPath := disks[0]
 		// Initialize FS object layer.
 		return newFSObjects(exportPath)
 	}
 	// Initialize XL object layer.
-	objAPI, err := newXLObjects(exportPaths)
+	objAPI, err := newXLObjects(disks, ignoredDisks)
 	if err == errXLWriteQuorum {
 		return objAPI, errors.New("Disks are different with last minio server run.")
 	}
@@ -41,11 +40,11 @@ func newObjectLayer(exportPaths []string) (ObjectLayer, error) {
 
 // configureServer handler returns final handler for the http server.
 func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler {
-	objAPI, err := newObjectLayer(srvCmdConfig.exportPaths)
+	objAPI, err := newObjectLayer(srvCmdConfig.disks, srvCmdConfig.ignoredDisks)
 	fatalIf(err, "Unable to intialize object layer.")
 
 	// Initialize storage rpc server.
-	storageRPC, err := newRPCServer(srvCmdConfig.exportPaths[0]) // FIXME: should only have one path.
+	storageRPC, err := newRPCServer(srvCmdConfig.disks[0]) // FIXME: should only have one path.
 	fatalIf(err, "Unable to initialize storage RPC server.")
 
 	// Initialize API.
@@ -70,6 +69,8 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler {
 	// List of some generic handlers which are applied for all
 	// incoming requests.
 	var handlerFns = []HandlerFunc{
+		// Limits the number of concurrent http requests.
+		setRateLimitHandler,
 		// Redirect some pre-defined browser request paths to a static
 		// location prefix.
 		setBrowserRedirectHandler,

rpc-client.go

@@ -145,15 +145,16 @@ func (n networkStorage) DeleteVol(volume string) error {
 // File operations.
 
 // CreateFile - create file.
-func (n networkStorage) AppendFile(volume, path string, buffer []byte) (m int64, err error) {
+func (n networkStorage) AppendFile(volume, path string, buffer []byte) (err error) {
+	reply := GenericReply{}
 	if err = n.rpcClient.Call("Storage.AppendFileHandler", AppendFileArgs{
 		Vol:    volume,
 		Path:   path,
 		Buffer: buffer,
-	}, &m); err != nil {
-		return 0, toStorageErr(err)
+	}, &reply); err != nil {
+		return toStorageErr(err)
 	}
-	return m, nil
+	return nil
 }
 
 // StatFile - get latest Stat information for a file at path.
@@ -167,6 +168,20 @@ func (n networkStorage) StatFile(volume, path string) (fileInfo FileInfo, err er
 	return fileInfo, nil
 }
 
+// ReadAll - reads entire contents of the file at path until EOF, returns the
+// contents in a byte slice. Returns buf == nil if err != nil.
+// This API is meant to be used on files which have small memory footprint, do
+// not use this on large files as it would cause server to crash.
+func (n networkStorage) ReadAll(volume, path string) (buf []byte, err error) {
+	if err = n.rpcClient.Call("Storage.ReadAllHandler", ReadAllArgs{
+		Vol:  volume,
+		Path: path,
+	}, &buf); err != nil {
+		return nil, toStorageErr(err)
+	}
+	return buf, nil
+}
+
 // ReadFile - reads a file.
 func (n networkStorage) ReadFile(volume string, path string, offset int64, buffer []byte) (m int64, err error) {
 	if err = n.rpcClient.Call("Storage.ReadFileHandler", ReadFileArgs{

rpc-server-datatypes.go

@@ -16,54 +16,92 @@
 
 package main
 
-// GenericReply generic rpc reply.
+// GenericReply represents any generic RPC reply.
 type GenericReply struct{}
 
-// GenericArgs generic rpc args.
+// GenericArgs represents any generic RPC arguments.
 type GenericArgs struct{}
 
-// ListVolsReply list vols rpc reply.
+// ListVolsReply represents list of vols RPC reply.
 type ListVolsReply struct {
+	// List of volumes stat information.
 	Vols []VolInfo
 }
 
-// ReadFileArgs contains read file arguments.
+// ReadAllArgs represents read all RPC arguments.
+type ReadAllArgs struct {
+	// Name of the volume.
+	Vol string
+
+	// Name of the path.
+	Path string
+}
+
+// ReadFileArgs represents read file RPC arguments.
 type ReadFileArgs struct {
-	Vol    string
-	Path   string
+	// Name of the volume.
+	Vol string
+
+	// Name of the path.
+	Path string
+
+	// Starting offset to start reading into Buffer.
 	Offset int64
+
+	// Data buffer read from the path at offset.
 	Buffer []byte
 }
 
-// AppendFileArgs contains append file arguments.
+// AppendFileArgs represents append file RPC arguments.
 type AppendFileArgs struct {
-	Vol    string
-	Path   string
+	// Name of the volume.
+	Vol string
+
+	// Name of the path.
+	Path string
+
+	// Data buffer to be saved at path.
 	Buffer []byte
 }
 
-// StatFileArgs contains stat file arguments.
+// StatFileArgs represents stat file RPC arguments.
 type StatFileArgs struct {
-	Vol  string
+	// Name of the volume.
+	Vol string
+
+	// Name of the path.
 	Path string
 }
 
-// DeleteFileArgs contains delete file arguments.
+// DeleteFileArgs represents delete file RPC arguments.
 type DeleteFileArgs struct {
-	Vol  string
+	// Name of the volume.
+	Vol string
+
+	// Name of the path.
 	Path string
 }
 
-// ListDirArgs contains list dir arguments.
+// ListDirArgs represents list contents RPC arguments.
 type ListDirArgs struct {
-	Vol  string
+	// Name of the volume.
+	Vol string
+
+	// Name of the path.
 	Path string
 }
 
-// RenameFileArgs contains rename file arguments.
+// RenameFileArgs represents rename file RPC arguments.
 type RenameFileArgs struct {
-	SrcVol  string
+	// Name of source volume.
+	SrcVol string
+
+	// Source path to be renamed.
 	SrcPath string
-	DstVol  string
+
+	// Name of destination volume.
+	DstVol string
+
+	// Destination path of renamed file.
 	DstPath string
 }

rpc-server.go

@@ -75,6 +75,16 @@ func (s *storageServer) ListDirHandler(arg *ListDirArgs, reply *[]string) error
 	return nil
 }
 
+// ReadAllHandler - read all handler is rpc wrapper to read all storage API.
+func (s *storageServer) ReadAllHandler(arg *ReadFileArgs, reply *[]byte) error {
+	buf, err := s.storage.ReadAll(arg.Vol, arg.Path)
+	if err != nil {
+		return err
+	}
+	reply = &buf
+	return nil
+}
+
 // ReadFileHandler - read file handler is rpc wrapper to read file.
 func (s *storageServer) ReadFileHandler(arg *ReadFileArgs, reply *int64) error {
 	n, err := s.storage.ReadFile(arg.Vol, arg.Path, arg.Offset, arg.Buffer)
@@ -86,31 +96,18 @@ func (s *storageServer) ReadFileHandler(arg *ReadFileArgs, reply *int64) error {
 }
 
 // AppendFileHandler - append file handler is rpc wrapper to append file.
-func (s *storageServer) AppendFileHandler(arg *AppendFileArgs, reply *int64) error {
-	n, err := s.storage.AppendFile(arg.Vol, arg.Path, arg.Buffer)
-	if err != nil {
-		return err
-	}
-	reply = &n
-	return nil
+func (s *storageServer) AppendFileHandler(arg *AppendFileArgs, reply *GenericReply) error {
+	return s.storage.AppendFile(arg.Vol, arg.Path, arg.Buffer)
 }
 
 // DeleteFileHandler - delete file handler is rpc wrapper to delete file.
 func (s *storageServer) DeleteFileHandler(arg *DeleteFileArgs, reply *GenericReply) error {
-	err := s.storage.DeleteFile(arg.Vol, arg.Path)
-	if err != nil {
-		return err
-	}
-	return nil
+	return s.storage.DeleteFile(arg.Vol, arg.Path)
 }
 
 // RenameFileHandler - rename file handler is rpc wrapper to rename file.
 func (s *storageServer) RenameFileHandler(arg *RenameFileArgs, reply *GenericReply) error {
-	err := s.storage.RenameFile(arg.SrcVol, arg.SrcPath, arg.DstVol, arg.DstPath)
-	if err != nil {
-		return err
-	}
-	return nil
+	return s.storage.RenameFile(arg.SrcVol, arg.SrcPath, arg.DstVol, arg.DstPath)
 }
 
 // Initialize new storage rpc.

runtime-checks.go

@@ -18,38 +18,12 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
-	"os"
 	"runtime"
-	"strings"
 
 	"github.com/hashicorp/go-version"
 	"github.com/minio/mc/pkg/console"
 )
 
-// isContainerized returns true if we are inside a containerized environment.
-func isContainerized() bool {
-	// Docker containers contain ".dockerenv" at their root path.
-	if _, e := os.Stat("/.dockerenv"); e == nil {
-		return true
-	}
-
-	// Check if cgroup policies for init process contains docker string.
-	if cgroupData, e := ioutil.ReadFile("/proc/1/cgroup"); e == nil {
-		if strings.Contains(string(cgroupData), "/docker-") {
-			return true
-		}
-	}
-
-	// Check if env var explicitly set
-	if allow := os.Getenv("ALLOW_CONTAINER_ROOT"); allow == "1" || strings.ToLower(allow) == "true" {
-		return true
-	}
-
-	/* Add checks for non-docker containers here. */
-	return false
-}
-
 // check if minimum Go version is met.
 func checkGoVersion() {
 	// Current version.