server: Move all the top level files into cmd folder. (#2490)

This change brings a change which was done for the 'mc'
package to allow for clean repo and have a cleaner
github drop in experience.

cmd/access-key.go

@@ -0,0 +1,86 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"regexp"
+)
+
+// credential container for access and secret keys.
+type credential struct {
+	AccessKeyID     string `json:"accessKey"`
+	SecretAccessKey string `json:"secretKey"`
+}
+
+const (
+	minioAccessID = 20
+	minioSecretID = 40
+)
+
+// isValidSecretKey - validate secret key.
+var isValidSecretKey = regexp.MustCompile(`^.{8,40}$`)
+
+// isValidAccessKey - validate access key.
+var isValidAccessKey = regexp.MustCompile(`^[a-zA-Z0-9\\-\\.\\_\\~]{5,20}$`)
+
+// mustGenAccessKeys - must generate access credentials.
+func mustGenAccessKeys() (creds credential) {
+	creds, err := genAccessKeys()
+	fatalIf(err, "Unable to generate access keys.")
+	return creds
+}
+
+// genAccessKeys - generate access credentials.
+func genAccessKeys() (credential, error) {
+	accessKeyID, err := genAccessKeyID()
+	if err != nil {
+		return credential{}, err
+	}
+	secretAccessKey, err := genSecretAccessKey()
+	if err != nil {
+		return credential{}, err
+	}
+	creds := credential{
+		AccessKeyID:     string(accessKeyID),
+		SecretAccessKey: string(secretAccessKey),
+	}
+	return creds, nil
+}
+
+// genAccessKeyID - generate random alpha numeric value using only uppercase characters
+// takes input as size in integer
+func genAccessKeyID() ([]byte, error) {
+	alpha := make([]byte, minioAccessID)
+	if _, err := rand.Read(alpha); err != nil {
+		return nil, err
+	}
+	for i := 0; i < minioAccessID; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return alpha, nil
+}
+
+// genSecretAccessKey - generate random base64 numeric value from a random seed.
+func genSecretAccessKey() ([]byte, error) {
+	rb := make([]byte, minioSecretID)
+	if _, err := rand.Read(rb); err != nil {
+		return nil, err
+	}
+	return []byte(base64.StdEncoding.EncodeToString(rb))[:minioSecretID], nil
+}

cmd/api-datatypes.go

@@ -0,0 +1,41 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+)
+
+// ObjectIdentifier carries key name for the object to delete.
+type ObjectIdentifier struct {
+	ObjectName string `xml:"Key"`
+}
+
+// createBucketConfiguration container for bucket configuration request from client.
+// Used for parsing the location from the request body for MakeBucketbucket.
+type createBucketLocationConfiguration struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"`
+	Location string   `xml:"LocationConstraint"`
+}
+
+// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
+type DeleteObjectsRequest struct {
+	// Element to enable quiet mode for the request
+	Quiet bool
+	// List of objects to be deleted
+	Objects []ObjectIdentifier `xml:"Object"`
+}

cmd/api-errors.go

@@ -0,0 +1,628 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+)
+
+// APIError structure
+type APIError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+// APIErrorResponse - error response format
+type APIErrorResponse struct {
+	XMLName    xml.Name `xml:"Error" json:"-"`
+	Code       string
+	Message    string
+	Key        string
+	BucketName string
+	Resource   string
+	RequestID  string `xml:"RequestId"`
+	HostID     string `xml:"HostId"`
+}
+
+// APIErrorCode type of error status.
+type APIErrorCode int
+
+// Error codes, non exhaustive list - http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+const (
+	ErrNone APIErrorCode = iota
+	ErrAccessDenied
+	ErrBadDigest
+	ErrBucketAlreadyExists
+	ErrEntityTooSmall
+	ErrEntityTooLarge
+	ErrIncompleteBody
+	ErrInternalError
+	ErrInvalidAccessKeyID
+	ErrInvalidBucketName
+	ErrInvalidDigest
+	ErrInvalidRange
+	ErrInvalidMaxKeys
+	ErrInvalidMaxUploads
+	ErrInvalidMaxParts
+	ErrInvalidPartNumberMarker
+	ErrInvalidRequestBody
+	ErrInvalidCopySource
+	ErrInvalidCopyDest
+	ErrInvalidPolicyDocument
+	ErrMalformedXML
+	ErrMissingContentLength
+	ErrMissingContentMD5
+	ErrMissingRequestBodyError
+	ErrNoSuchBucket
+	ErrNoSuchBucketPolicy
+	ErrNoSuchKey
+	ErrNoSuchUpload
+	ErrNotImplemented
+	ErrPreconditionFailed
+	ErrRequestTimeTooSkewed
+	ErrSignatureDoesNotMatch
+	ErrMethodNotAllowed
+	ErrInvalidPart
+	ErrInvalidPartOrder
+	ErrAuthorizationHeaderMalformed
+	ErrMalformedPOSTRequest
+	ErrSignatureVersionNotSupported
+	ErrBucketNotEmpty
+	ErrAllAccessDisabled
+	ErrMalformedPolicy
+	ErrMissingFields
+	ErrMissingCredTag
+	ErrCredMalformed
+	ErrInvalidRegion
+	ErrInvalidService
+	ErrInvalidRequestVersion
+	ErrMissingSignTag
+	ErrMissingSignHeadersTag
+	ErrPolicyAlreadyExpired
+	ErrMalformedDate
+	ErrMalformedPresignedDate
+	ErrMalformedCredentialDate
+	ErrMalformedCredentialRegion
+	ErrMalformedExpires
+	ErrNegativeExpires
+	ErrAuthHeaderEmpty
+	ErrExpiredPresignRequest
+	ErrUnsignedHeaders
+	ErrMissingDateHeader
+	ErrInvalidQuerySignatureAlgo
+	ErrInvalidQueryParams
+	ErrBucketAlreadyOwnedByYou
+	// Add new error codes here.
+
+	// Bucket notification related errors.
+	ErrEventNotification
+	ErrARNNotification
+	ErrRegionNotification
+	ErrOverlappingFilterNotification
+	ErrFilterNameInvalid
+	ErrFilterNamePrefix
+	ErrFilterNameSuffix
+	ErrFilterValueInvalid
+
+	// S3 extended errors.
+	ErrContentSHA256Mismatch
+
+	// Add new extended error codes here.
+
+	// Minio extended errors.
+	ErrReadQuorum
+	ErrWriteQuorum
+	ErrStorageFull
+	ErrObjectExistsAsDirectory
+	ErrPolicyNesting
+	ErrInvalidObjectName
+	// Add new extended error codes here.
+	// Please open a https://github.com/minio/minio/issues before adding
+	// new error codes here.
+)
+
+// error code to APIError structure, these fields carry respective
+// descriptions for all the error responses.
+var errorCodeResponse = map[APIErrorCode]APIError{
+	ErrInvalidCopyDest: {
+		Code:           "InvalidRequest",
+		Description:    "This copy request is illegal because it is trying to copy an object to itself.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCopySource: {
+		Code:           "InvalidArgument",
+		Description:    "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRequestBody: {
+		Code:           "InvalidArgument",
+		Description:    "Body shouldn't be set for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxUploads: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-uploads must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxKeys: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxParts: {
+		Code:           "InvalidArgument",
+		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPartNumberMarker: {
+		Code:           "InvalidArgument",
+		Description:    "Argument partNumberMarker must be an integer.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPolicyDocument: {
+		Code:           "InvalidPolicyDocument",
+		Description:    "The content of the form does not meet the conditions specified in the policy document.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAccessDenied: {
+		Code:           "AccessDenied",
+		Description:    "Access Denied.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrBadDigest: {
+		Code:           "BadDigest",
+		Description:    "The Content-Md5 you specified did not match what we received.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketAlreadyExists: {
+		Code:           "BucketAlreadyExists",
+		Description:    "The requested bucket name is not available.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrEntityTooSmall: {
+		Code:           "EntityTooSmall",
+		Description:    "Your proposed upload is smaller than the minimum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrEntityTooLarge: {
+		Code:           "EntityTooLarge",
+		Description:    "Your proposed upload exceeds the maximum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrIncompleteBody: {
+		Code:           "IncompleteBody",
+		Description:    "You did not provide the number of bytes specified by the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInternalError: {
+		Code:           "InternalError",
+		Description:    "We encountered an internal error, please try again.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	ErrInvalidAccessKeyID: {
+		Code:           "InvalidAccessKeyID",
+		Description:    "The access key ID you provided does not exist in our records.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrInvalidBucketName: {
+		Code:           "InvalidBucketName",
+		Description:    "The specified bucket is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidDigest: {
+		Code:           "InvalidDigest",
+		Description:    "The Content-Md5 you specified is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRange: {
+		Code:           "InvalidRange",
+		Description:    "The requested range is not satisfiable",
+		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
+	},
+	ErrMalformedXML: {
+		Code:           "MalformedXML",
+		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingContentLength: {
+		Code:           "MissingContentLength",
+		Description:    "You must provide the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
+	ErrMissingContentMD5: {
+		Code:           "MissingContentMD5",
+		Description:    "Missing required header for this request: Content-Md5.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingRequestBodyError: {
+		Code:           "MissingRequestBodyError",
+		Description:    "Request body is empty.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
+	ErrNoSuchBucket: {
+		Code:           "NoSuchBucket",
+		Description:    "The specified bucket does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchBucketPolicy: {
+		Code:           "NoSuchBucketPolicy",
+		Description:    "The specified bucket does not have a bucket policy.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchKey: {
+		Code:           "NoSuchKey",
+		Description:    "The specified key does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchUpload: {
+		Code:           "NoSuchUpload",
+		Description:    "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNotImplemented: {
+		Code:           "NotImplemented",
+		Description:    "A header you provided implies functionality that is not implemented",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+	ErrPreconditionFailed: {
+		Code:           "PreconditionFailed",
+		Description:    "At least one of the pre-conditions you specified did not hold",
+		HTTPStatusCode: http.StatusPreconditionFailed,
+	},
+	ErrRequestTimeTooSkewed: {
+		Code:           "RequestTimeTooSkewed",
+		Description:    "The difference between the request time and the server's time is too large.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "The request signature we calculated does not match the signature you provided. Check your key and signing method.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMethodNotAllowed: {
+		Code:           "MethodNotAllowed",
+		Description:    "The specified method is not allowed against this resource.",
+		HTTPStatusCode: http.StatusMethodNotAllowed,
+	},
+	ErrInvalidPart: {
+		Code:           "InvalidPart",
+		Description:    "One or more of the specified parts could not be found.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPartOrder: {
+		Code:           "InvalidPartOrder",
+		Description:    "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthorizationHeaderMalformed: {
+		Code:           "AuthorizationHeaderMalformed",
+		Description:    "The authorization header is malformed; the region is wrong; expecting 'us-east-1'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedPOSTRequest: {
+		Code:           "MalformedPOSTRequest",
+		Description:    "The body of your POST request is not well-formed multipart/form-data.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSignatureVersionNotSupported: {
+		Code:           "InvalidRequest",
+		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrAllAccessDisabled: {
+		Code:           "AllAccessDisabled",
+		Description:    "All access to this bucket has been disabled.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMalformedPolicy: {
+		Code:           "MalformedPolicy",
+		Description:    "Policy has invalid resource.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingFields: {
+		Code:           "MissingFields",
+		Description:    "Missing fields in request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingCredTag: {
+		Code:           "InvalidRequest",
+		Description:    "Missing Credential field for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrCredMalformed: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedDate: {
+		Code:           "MalformedDate",
+		Description:    "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedPresignedDate: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
+	// right Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format \"%s\". This date in the credential must be in the format \"yyyyMMdd\".",
+	// Need changes to make sure variable messages can be constructed.
+	ErrMalformedCredentialDate: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format \"%s\". This date in the credential must be in the format \"yyyyMMdd\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
+	// right Description:    "Error parsing the X-Amz-Credential parameter; the region 'us-east-' is wrong; expecting 'us-east-1'".
+	// Need changes to make sure variable messages can be constructed.
+	ErrMalformedCredentialRegion: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; the region is wrong;",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRegion: {
+		Code:           "InvalidRegion",
+		Description:    "Region does not match.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
+	// right Description:   "Error parsing the X-Amz-Credential parameter; incorrect service \"s4\". This endpoint belongs to \"s3\".".
+	// Need changes to make sure variable messages can be constructed.
+	ErrInvalidService: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
+	// Description:   "Error parsing the X-Amz-Credential parameter; incorrect terminal "aws4_reque". This endpoint uses "aws4_request".
+	// Need changes to make sure variable messages can be constructed.
+	ErrInvalidRequestVersion: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Error parsing the X-Amz-Credential parameter; incorrect terminal. This endpoint uses \"aws4_request\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignTag: {
+		Code:           "AccessDenied",
+		Description:    "Signature header missing Signature field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignHeadersTag: {
+		Code:           "InvalidArgument",
+		Description:    "Signature header missing SignedHeaders field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPolicyAlreadyExpired: {
+		Code:           "AccessDenied",
+		Description:    "Invalid according to Policy: Policy expired.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires should be a number",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrNegativeExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires must be non-negative",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthHeaderEmpty: {
+		Code:           "InvalidArgument",
+		Description:    "Authorization header is invalid -- one and only one ' ' (space) required.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingDateHeader: {
+		Code:           "AccessDenied",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQuerySignatureAlgo: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrExpiredPresignRequest: {
+		Code:           "AccessDenied",
+		Description:    "Request has expired",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// FIXME: Actual XML error response also contains the header which missed in lsit of signed header parameters.
+	ErrUnsignedHeaders: {
+		Code:           "AccessDenied",
+		Description:    "There were headers present in the request which were not signed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQueryParams: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketAlreadyOwnedByYou: {
+		Code:           "BucketAlreadyOwnedByYou",
+		Description:    "Your previous request to create the named bucket succeeded and you already own it.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+
+	/// Bucket notification related errors.
+	ErrEventNotification: {
+		Code:           "InvalidArgument",
+		Description:    "A specified event is not supported for notifications.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrARNNotification: {
+		Code:           "InvalidArgument",
+		Description:    "A specified destination ARN does not exist or is not well-formed. Verify the destination ARN.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrRegionNotification: {
+		Code:           "InvalidArgument",
+		Description:    "A specified destination is in a different region than the bucket. You must use a destination that resides in the same region as the bucket.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOverlappingFilterNotification: {
+		Code:           "InvalidArgument",
+		Description:    "An object key name filtering rule defined with overlapping prefixes, overlapping suffixes, or overlapping combinations of prefixes and suffixes for the same event types.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrFilterNameInvalid: {
+		Code:           "InvalidArgument",
+		Description:    "filter rule name must be either prefix or suffix",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrFilterNamePrefix: {
+		Code:           "InvalidArgument",
+		Description:    "Cannot specify more than one prefix rule in a filter.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrFilterNameSuffix: {
+		Code:           "InvalidArgument",
+		Description:    "Cannot specify more than one suffix rule in a filter.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrFilterValueInvalid: {
+		Code:           "InvalidArgument",
+		Description:    "Size of filter rule value cannot exceed 1024 bytes in UTF-8 representation",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
+	/// S3 extensions.
+	ErrContentSHA256Mismatch: {
+		Code:           "XAmzContentSHA256Mismatch",
+		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
+	/// Minio extensions.
+	ErrStorageFull: {
+		Code:           "XMinioStorageFull",
+		Description:    "Storage backend has reached its minimum free disk threshold. Please delete few objects to proceed.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	ErrObjectExistsAsDirectory: {
+		Code:           "XMinioObjectExistsAsDirectory",
+		Description:    "Object name already exists as a directory.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrReadQuorum: {
+		Code:           "XMinioReadQuorum",
+		Description:    "Multiple disk failures, unable to reconstruct data.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrWriteQuorum: {
+		Code:           "XMinioWriteQuorum",
+		Description:    "Multiple disks failures, unable to write data.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrPolicyNesting: {
+		Code:           "XMinioPolicyNesting",
+		Description:    "Policy nesting conflict has occurred.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrInvalidObjectName: {
+		Code:           "XMinioInvalidObjectName",
+		Description:    "Object name contains unsupported characters. Unsupported characters are `^*|\\\"",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// Add your error structure here.
+}
+
+// toAPIErrorCode - Converts embedded errors. Convenience
+// function written to handle all cases where we have known types of
+// errors returned by underlying layers.
+func toAPIErrorCode(err error) (apiErr APIErrorCode) {
+	if err == nil {
+		return ErrNone
+	}
+	// Verify if the underlying error is signature mismatch.
+	switch err {
+	case errSignatureMismatch:
+		apiErr = ErrSignatureDoesNotMatch
+	case errContentSHA256Mismatch:
+		apiErr = ErrContentSHA256Mismatch
+	}
+	if apiErr != ErrNone {
+		// If there was a match in the above switch case.
+		return apiErr
+	}
+	switch err.(type) {
+	case StorageFull:
+		apiErr = ErrStorageFull
+	case BadDigest:
+		apiErr = ErrBadDigest
+	case IncompleteBody:
+		apiErr = ErrIncompleteBody
+	case ObjectExistsAsDirectory:
+		apiErr = ErrObjectExistsAsDirectory
+	case BucketNameInvalid:
+		apiErr = ErrInvalidBucketName
+	case BucketNotFound:
+		apiErr = ErrNoSuchBucket
+	case BucketNotEmpty:
+		apiErr = ErrBucketNotEmpty
+	case BucketExists:
+		apiErr = ErrBucketAlreadyOwnedByYou
+	case ObjectNotFound:
+		apiErr = ErrNoSuchKey
+	case ObjectNameInvalid:
+		apiErr = ErrInvalidObjectName
+	case InvalidUploadID:
+		apiErr = ErrNoSuchUpload
+	case InvalidPart:
+		apiErr = ErrInvalidPart
+	case InsufficientWriteQuorum:
+		apiErr = ErrWriteQuorum
+	case InsufficientReadQuorum:
+		apiErr = ErrReadQuorum
+	case PartTooSmall:
+		apiErr = ErrEntityTooSmall
+	default:
+		apiErr = ErrInternalError
+	}
+	return apiErr
+}
+
+// getAPIError provides API Error for input API error code.
+func getAPIError(code APIErrorCode) APIError {
+	return errorCodeResponse[code]
+}
+
+// getErrorResponse gets in standard error and resource value and
+// provides a encodable populated response values
+func getAPIErrorResponse(err APIError, resource string) APIErrorResponse {
+	var data = APIErrorResponse{}
+	data.Code = err.Code
+	data.Message = err.Description
+	if resource != "" {
+		data.Resource = resource
+	}
+	// TODO implement this in future
+	data.RequestID = "3L137"
+	data.HostID = "3L137"
+
+	return data
+}
+
+func getErrMalformedCredentialDate(malformedDateStr string) {
+
+}

cmd/api-headers.go

@@ -0,0 +1,89 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/xml"
+	"net/http"
+	"runtime"
+	"strconv"
+)
+
+//// helpers
+
+// Static alphaNumeric table used for generating unique request ids
+var alphaNumericTable = []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+// generateRequestID - Generate request id
+func generateRequestID() []byte {
+	alpha := make([]byte, 16)
+	rand.Read(alpha)
+	for i := 0; i < 16; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return alpha
+}
+
+// Write http common headers
+func setCommonHeaders(w http.ResponseWriter) {
+	// Set unique request ID for each reply.
+	w.Header().Set("X-Amz-Request-Id", string(generateRequestID()))
+	w.Header().Set("Server", ("Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"))
+	w.Header().Set("Accept-Ranges", "bytes")
+}
+
+// Encodes the response headers into XML format.
+func encodeResponse(response interface{}) []byte {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	e.Encode(response)
+	return bytesBuffer.Bytes()
+}
+
+// Write object header
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
+	// set common headers
+	setCommonHeaders(w)
+
+	// Set content length.
+	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
+
+	// Set last modified time.
+	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
+	w.Header().Set("Last-Modified", lastModified)
+
+	// Set Etag if available.
+	if objInfo.MD5Sum != "" {
+		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+	}
+
+	// Set all other user defined metadata.
+	for k, v := range objInfo.UserDefined {
+		w.Header().Set(k, v)
+	}
+
+	// for providing ranged content
+	if contentRange != nil && contentRange.offsetBegin > -1 {
+		// Override content-length
+		w.Header().Set("Content-Length", strconv.FormatInt(contentRange.getLength(), 10))
+		w.Header().Set("Content-Range", contentRange.String())
+		w.WriteHeader(http.StatusPartialContent)
+	}
+}

cmd/api-headers_test.go

@@ -0,0 +1,40 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"testing"
+)
+
+func TestGenerateRequestID(t *testing.T) {
+	// Ensure that it returns an alphanumeric result of length 16.
+	var id = generateRequestID()
+
+	if len(id) != 16 {
+		t.Fail()
+	}
+
+	var e rune
+	for _, char := range id {
+		e = rune(char)
+
+		// Ensure that it is alphanumeric, in this case, between 0-9 and A-Z.
+		if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) {
+			t.Fail()
+		}
+	}
+}

cmd/api-resources.go

@@ -0,0 +1,79 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/url"
+	"strconv"
+)
+
+// Parse bucket url queries
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse bucket url queries for ListObjects V2.
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, maxkeys int, encodingType string) {
+	prefix = values.Get("prefix")
+	startAfter = values.Get("start-after")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
+	encodingType = values.Get("encoding-type")
+	token = values.Get("continuation-token")
+	return
+}
+
+// Parse bucket url queries for ?uploads
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+	prefix = values.Get("prefix")
+	keyMarker = values.Get("key-marker")
+	uploadIDMarker = values.Get("upload-id-marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-uploads") != "" {
+		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
+	} else {
+		maxUploads = maxUploadsList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse object url queries
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
+	uploadID = values.Get("uploadId")
+	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+	if values.Get("max-parts") != "" {
+		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+	} else {
+		maxParts = maxPartsList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}

cmd/api-response-multipart.go

@@ -0,0 +1,54 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package cmd file carries any specific responses constructed/necessary in
+// multipart operations.
+package cmd
+
+import "net/http"
+
+// writeErrorResponsePartTooSmall - function is used specifically to
+// construct a proper error response during CompleteMultipartUpload
+// when one of the parts is < 5MB.
+// The requirement comes due to the fact that generic ErrorResponse
+// XML doesn't carry the additional fields required to send this
+// error. So we construct a new type which lies well within the scope
+// of this function.
+func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
+	// Represents additional fields necessary for ErrPartTooSmall S3 error.
+	type completeMultipartAPIError struct {
+		// Proposed size represents uploaded size of the part.
+		ProposedSize int64
+		// Minimum size allowed epresents the minimum size allowed per
+		// part. Defaults to 5MB.
+		MinSizeAllowed int64
+		// Part number of the part which is incorrect.
+		PartNumber int
+		// ETag of the part which is incorrect.
+		PartETag string
+		// Other default XML error responses.
+		APIErrorResponse
+	}
+	// Generate complete multipart error response.
+	errorResponse := getAPIErrorResponse(getAPIError(toAPIErrorCode(err)), r.URL.Path)
+	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
+	encodedErrorResponse := encodeResponse(cmpErrResp)
+	// Write error body
+	w.Write(encodedErrorResponse)
+	w.(http.Flusher).Flush()
+}
+
+// Add any other multipart specific responses here.

cmd/api-response.go

@@ -0,0 +1,519 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+	"path"
+	"time"
+)
+
+const (
+	timeFormatAMZ  = "2006-01-02T15:04:05.000Z" // Reply date format
+	maxObjectList  = 1000                       // Limit number of objects in a listObjectsResponse.
+	maxUploadsList = 1000                       // Limit number of uploads in a listUploadsResponse.
+	maxPartsList   = 1000                       // Limit number of parts in a listPartsResponse.
+)
+
+// LocationResponse - format for location response.
+type LocationResponse struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
+	Location string   `xml:",chardata"`
+}
+
+// ListObjectsResponse - format for list objects response.
+type ListObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+
+	CommonPrefixes []CommonPrefix
+	Contents       []Object
+
+	Delimiter string
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string
+
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+	Marker      string
+	MaxKeys     int
+	Name        string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	NextMarker string
+	Prefix     string
+}
+
+// ListObjectsV2Response - format for list objects response.
+type ListObjectsV2Response struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+
+	CommonPrefixes []CommonPrefix
+	Contents       []Object
+
+	Delimiter string
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string
+
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+	StartAfter  string
+	MaxKeys     int
+	Name        string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	ContinuationToken     string
+	NextContinuationToken string
+	Prefix                string
+}
+
+// Part container for part metadata.
+type Part struct {
+	PartNumber   int
+	ETag         string
+	LastModified string
+	Size         int64
+}
+
+// ListPartsResponse - format for list parts response.
+type ListPartsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+
+	Initiator Initiator
+	Owner     Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	// List of parts.
+	Parts []Part `xml:"Part"`
+}
+
+// ListMultipartUploadsResponse - format for list multipart uploads response.
+type ListMultipartUploadsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
+
+	Bucket             string
+	KeyMarker          string
+	UploadIDMarker     string `xml:"UploadIdMarker"`
+	NextKeyMarker      string
+	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
+	EncodingType       string
+	MaxUploads         int
+	IsTruncated        bool
+	Uploads            []Upload `xml:"Upload"`
+	Prefix             string
+	Delimiter          string
+	CommonPrefixes     []CommonPrefix
+}
+
+// ListBucketsResponse - format for list buckets response
+type ListBucketsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
+	// Container for one or more buckets.
+	Buckets struct {
+		Buckets []Bucket `xml:"Bucket"`
+	} // Buckets are nested
+	Owner Owner
+}
+
+// Upload container for in progress multipart upload
+type Upload struct {
+	Key          string
+	UploadID     string `xml:"UploadId"`
+	Initiator    Initiator
+	Owner        Owner
+	StorageClass string
+	Initiated    string
+}
+
+// CommonPrefix container for prefix response in ListObjectsResponse
+type CommonPrefix struct {
+	Prefix string
+}
+
+// Bucket container for bucket metadata
+type Bucket struct {
+	Name         string
+	CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
+}
+
+// Object container for object metadata
+type Object struct {
+	ETag         string
+	Key          string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+	Size         int64
+
+	Owner Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+}
+
+// CopyObjectResponse container returns ETag and LastModified of the
+// successfully copied object
+type CopyObjectResponse struct {
+	XMLName      xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
+	ETag         string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+}
+
+// Initiator inherit from Owner struct, fields are same
+type Initiator Owner
+
+// Owner - bucket owner/principal
+type Owner struct {
+	ID          string
+	DisplayName string
+}
+
+// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
+type InitiateMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+}
+
+// CompleteMultipartUploadResponse container for completed multipart upload response
+type CompleteMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
+
+	Location string
+	Bucket   string
+	Key      string
+	ETag     string
+}
+
+// PostResponse container for completed post upload response
+type PostResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PostResponse" json:"-"`
+
+	Location string
+	Bucket   string
+	Key      string
+	ETag     string
+}
+
+// DeleteError structure.
+type DeleteError struct {
+	Code    string
+	Message string
+	Key     string
+}
+
+// DeleteObjectsResponse container for multiple object deletes.
+type DeleteObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`
+
+	// Collection of all deleted objects
+	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`
+
+	// Collection of errors deleting certain objects.
+	Errors []DeleteError `xml:"Error,omitempty"`
+}
+
+// getLocation get URL location.
+func getLocation(r *http.Request) string {
+	return path.Clean(r.URL.Path) // Clean any trailing slashes.
+}
+
+// getObjectLocation gets the relative URL for an object
+func getObjectLocation(bucketName string, key string) string {
+	return "/" + bucketName + "/" + key
+}
+
+// takes an array of Bucketmetadata information for serialization
+// input:
+// array of bucket metadata
+//
+// output:
+// populated struct that can be serialized to match xml and json api spec output
+func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
+	var listbuckets []Bucket
+	var data = ListBucketsResponse{}
+	var owner = Owner{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, bucket := range buckets {
+		var listbucket = Bucket{}
+		listbucket.Name = bucket.Name
+		listbucket.CreationDate = bucket.Created.Format(timeFormatAMZ)
+		listbuckets = append(listbuckets, listbucket)
+	}
+
+	data.Owner = owner
+	data.Buckets.Buckets = listbuckets
+
+	return data
+}
+
+// generates an ListObjectsV1 response for the said bucket with other enumerated options.
+func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
+	var contents []Object
+	var prefixes []CommonPrefix
+	var owner = Owner{}
+	var data = ListObjectsResponse{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, object := range resp.Objects {
+		var content = Object{}
+		if object.Name == "" {
+			continue
+		}
+		content.Key = object.Name
+		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZ)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
+		}
+		content.Size = object.Size
+		content.StorageClass = "STANDARD"
+		content.Owner = owner
+		contents = append(contents, content)
+	}
+	// TODO - support EncodingType in xml decoding
+	data.Name = bucket
+	data.Contents = contents
+
+	data.Prefix = prefix
+	data.Marker = marker
+	data.Delimiter = delimiter
+	data.MaxKeys = maxKeys
+
+	data.NextMarker = resp.NextMarker
+	data.IsTruncated = resp.IsTruncated
+	for _, prefix := range resp.Prefixes {
+		var prefixItem = CommonPrefix{}
+		prefixItem.Prefix = prefix
+		prefixes = append(prefixes, prefixItem)
+	}
+	data.CommonPrefixes = prefixes
+	return data
+}
+
+// generates an ListObjectsV2 response for the said bucket with other enumerated options.
+func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
+	var contents []Object
+	var prefixes []CommonPrefix
+	var owner = Owner{}
+	var data = ListObjectsV2Response{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, object := range resp.Objects {
+		var content = Object{}
+		if object.Name == "" {
+			continue
+		}
+		content.Key = object.Name
+		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZ)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
+		}
+		content.Size = object.Size
+		content.StorageClass = "STANDARD"
+		content.Owner = owner
+		contents = append(contents, content)
+	}
+	// TODO - support EncodingType in xml decoding
+	data.Name = bucket
+	data.Contents = contents
+
+	data.StartAfter = startAfter
+	data.Delimiter = delimiter
+	data.Prefix = prefix
+	data.MaxKeys = maxKeys
+	data.ContinuationToken = token
+	data.NextContinuationToken = resp.NextMarker
+	data.IsTruncated = resp.IsTruncated
+	for _, prefix := range resp.Prefixes {
+		var prefixItem = CommonPrefix{}
+		prefixItem.Prefix = prefix
+		prefixes = append(prefixes, prefixItem)
+	}
+	data.CommonPrefixes = prefixes
+	return data
+}
+
+// generateCopyObjectResponse
+func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
+	return CopyObjectResponse{
+		ETag:         "\"" + etag + "\"",
+		LastModified: lastModified.UTC().Format(timeFormatAMZ),
+	}
+}
+
+// generateInitiateMultipartUploadResponse
+func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse {
+	return InitiateMultipartUploadResponse{
+		Bucket:   bucket,
+		Key:      key,
+		UploadID: uploadID,
+	}
+}
+
+// generateCompleteMultipartUploadResponse
+func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
+	return CompleteMultipartUploadResponse{
+		Location: location,
+		Bucket:   bucket,
+		Key:      key,
+		ETag:     etag,
+	}
+}
+
+// generateListPartsResult
+func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
+	// TODO - support EncodingType in xml decoding
+	listPartsResponse := ListPartsResponse{}
+	listPartsResponse.Bucket = partsInfo.Bucket
+	listPartsResponse.Key = partsInfo.Object
+	listPartsResponse.UploadID = partsInfo.UploadID
+	listPartsResponse.StorageClass = "STANDARD"
+	listPartsResponse.Initiator.ID = "minio"
+	listPartsResponse.Initiator.DisplayName = "minio"
+	listPartsResponse.Owner.ID = "minio"
+	listPartsResponse.Owner.DisplayName = "minio"
+
+	listPartsResponse.MaxParts = partsInfo.MaxParts
+	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
+	listPartsResponse.IsTruncated = partsInfo.IsTruncated
+	listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
+
+	listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
+	for index, part := range partsInfo.Parts {
+		newPart := Part{}
+		newPart.PartNumber = part.PartNumber
+		newPart.ETag = "\"" + part.ETag + "\""
+		newPart.Size = part.Size
+		newPart.LastModified = part.LastModified.UTC().Format(timeFormatAMZ)
+		listPartsResponse.Parts[index] = newPart
+	}
+	return listPartsResponse
+}
+
+// generateListMultipartUploadsResponse
+func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse {
+	listMultipartUploadsResponse := ListMultipartUploadsResponse{}
+	listMultipartUploadsResponse.Bucket = bucket
+	listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter
+	listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated
+	listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType
+	listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix
+	listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker
+	listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker
+	listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads
+	listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker
+	listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker
+	listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes))
+	for index, commonPrefix := range multipartsInfo.CommonPrefixes {
+		listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{
+			Prefix: commonPrefix,
+		}
+	}
+	listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads))
+	for index, upload := range multipartsInfo.Uploads {
+		newUpload := Upload{}
+		newUpload.UploadID = upload.UploadID
+		newUpload.Key = upload.Object
+		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZ)
+		listMultipartUploadsResponse.Uploads[index] = newUpload
+	}
+	return listMultipartUploadsResponse
+}
+
+// generate multi objects delete response.
+func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier, errs []DeleteError) DeleteObjectsResponse {
+	deleteResp := DeleteObjectsResponse{}
+	if !quiet {
+		deleteResp.DeletedObjects = deletedObjects
+	}
+	deleteResp.Errors = errs
+	return deleteResp
+}
+
+// writeSuccessResponse write success headers and response if any.
+func writeSuccessResponse(w http.ResponseWriter, response []byte) {
+	setCommonHeaders(w)
+	if response == nil {
+		w.WriteHeader(http.StatusOK)
+		return
+	}
+	w.Write(response)
+	w.(http.Flusher).Flush()
+}
+
+// writeSuccessNoContent write success headers with http status 204
+func writeSuccessNoContent(w http.ResponseWriter) {
+	setCommonHeaders(w)
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// writeErrorRespone write error headers
+func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorCode APIErrorCode, resource string) {
+	error := getAPIError(errorCode)
+	// set common headers
+	setCommonHeaders(w)
+	// write Header
+	w.WriteHeader(error.HTTPStatusCode)
+	writeErrorResponseNoHeader(w, req, errorCode, resource)
+}
+
+func writeErrorResponseNoHeader(w http.ResponseWriter, req *http.Request, errorCode APIErrorCode, resource string) {
+	error := getAPIError(errorCode)
+	// Generate error response.
+	errorResponse := getAPIErrorResponse(error, resource)
+	encodedErrorResponse := encodeResponse(errorResponse)
+	// HEAD should have no body, do not attempt to write to it
+	if req.Method != "HEAD" {
+		// write error body
+		w.Write(encodedErrorResponse)
+		w.(http.Flusher).Flush()
+	}
+}

cmd/api-router.go

@@ -0,0 +1,94 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import router "github.com/gorilla/mux"
+
+// objectAPIHandler implements and provides http handlers for S3 API.
+type objectAPIHandlers struct {
+	ObjectAPI ObjectLayer
+}
+
+// registerAPIRouter - registers S3 compatible APIs.
+func registerAPIRouter(mux *router.Router, api objectAPIHandlers) {
+	// API Router
+	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
+
+	// Bucket router
+	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+
+	/// Object operations
+
+	// HeadObject
+	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
+	// PutObjectPart
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	// ListObjectPxarts
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
+	// CompleteMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// NewMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
+	// AbortMultipartUpload
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// GetObject
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
+	// CopyObject
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
+	// PutObject
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
+	// DeleteObject
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
+
+	/// Bucket operations
+
+	// GetBucketLocation
+	bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
+	// GetBucketPolicy
+	bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
+	// GetBucketNotification
+	bucket.Methods("GET").HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "")
+	// ListenBucketNotification
+	bucket.Methods("GET").HandlerFunc(api.ListenBucketNotificationHandler).Queries("notificationARN", "{notificationARN:.*}")
+	// ListMultipartUploads
+	bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
+	// ListObjectsV2
+	bucket.Methods("GET").HandlerFunc(api.ListObjectsV2Handler).Queries("list-type", "2")
+	// ListObjectsV1 (Legacy)
+	bucket.Methods("GET").HandlerFunc(api.ListObjectsV1Handler)
+	// PutBucketPolicy
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
+	// PutBucketNotification
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "")
+	// PutBucket
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
+	// HeadBucket
+	bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
+	// PostPolicy
+	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
+	// DeleteMultipleObjects
+	bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
+	// DeleteBucketPolicy
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
+	// DeleteBucket
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
+
+	/// Root operation
+
+	// ListBuckets
+	apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
+}

cmd/auth-handler.go

@@ -0,0 +1,203 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// Verify if the request http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD"
+func isRequestUnsignedPayload(r *http.Request) bool {
+	return r.Header.Get("x-amz-content-sha256") == unsignedPayload
+}
+
+// Verify if request has JWT.
+func isRequestJWT(r *http.Request) bool {
+	return strings.HasPrefix(r.Header.Get("Authorization"), jwtAlgorithm)
+}
+
+// Verify if request has AWS Signature Version '4'.
+func isRequestSignatureV4(r *http.Request) bool {
+	return strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm)
+}
+
+// Verify if request has AWS PreSign Version '4'.
+func isRequestPresignedSignatureV4(r *http.Request) bool {
+	_, ok := r.URL.Query()["X-Amz-Credential"]
+	return ok
+}
+
+// Verify if request has AWS Post policy Signature Version '4'.
+func isRequestPostPolicySignatureV4(r *http.Request) bool {
+	return strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") && r.Method == "POST"
+}
+
+// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
+func isRequestSignStreamingV4(r *http.Request) bool {
+	return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 && r.Method == "PUT"
+}
+
+// Authorization type.
+type authType int
+
+// List of all supported auth types.
+const (
+	authTypeUnknown authType = iota
+	authTypeAnonymous
+	authTypePresigned
+	authTypePostPolicy
+	authTypeStreamingSigned
+	authTypeSigned
+	authTypeJWT
+)
+
+// Get request authentication type.
+func getRequestAuthType(r *http.Request) authType {
+	if isRequestSignStreamingV4(r) {
+		return authTypeStreamingSigned
+	} else if isRequestSignatureV4(r) {
+		return authTypeSigned
+	} else if isRequestPresignedSignatureV4(r) {
+		return authTypePresigned
+	} else if isRequestJWT(r) {
+		return authTypeJWT
+	} else if isRequestPostPolicySignatureV4(r) {
+		return authTypePostPolicy
+	} else if _, ok := r.Header["Authorization"]; !ok {
+		return authTypeAnonymous
+	}
+	return authTypeUnknown
+}
+
+// sum256 calculate sha256 sum for an input byte array
+func sum256(data []byte) []byte {
+	hash := sha256.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// sumMD5 calculate md5 sum for an input byte array
+func sumMD5(data []byte) []byte {
+	hash := md5.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// Verify if request has valid AWS Signature Version '4'.
+func isReqAuthenticated(r *http.Request) (s3Error APIErrorCode) {
+	if r == nil {
+		return ErrInternalError
+	}
+	payload, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return ErrInternalError
+	}
+	// Verify Content-Md5, if payload is set.
+	if r.Header.Get("Content-Md5") != "" {
+		if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+			return ErrBadDigest
+		}
+	}
+	// Populate back the payload.
+	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+	validateRegion := true // Validate region.
+	var sha256sum string
+	// Skips calculating sha256 on the payload on server,
+	// if client requested for it.
+	if skipContentSha256Cksum(r) {
+		sha256sum = unsignedPayload
+	} else {
+		sha256sum = hex.EncodeToString(sum256(payload))
+	}
+	if isRequestSignatureV4(r) {
+		return doesSignatureMatch(sha256sum, r, validateRegion)
+	} else if isRequestPresignedSignatureV4(r) {
+		return doesPresignedSignatureMatch(sha256sum, r, validateRegion)
+	}
+	return ErrAccessDenied
+}
+
+// checkAuth - checks for conditions satisfying the authorization of
+// the incoming request. Request should be either Presigned or Signed
+// in accordance with AWS S3 Signature V4 requirements. ErrAccessDenied
+// is returned for unhandled auth type. Once the auth type is indentified
+// request headers and body are used to calculate the signature validating
+// the client signature present in request.
+func checkAuth(r *http.Request) APIErrorCode {
+	aType := getRequestAuthType(r)
+	if aType != authTypePresigned && aType != authTypeSigned {
+		// For all unhandled auth types return error AccessDenied.
+		return ErrAccessDenied
+	}
+	// Validates the request for both Presigned and Signed.
+	return isReqAuthenticated(r)
+}
+
+// authHandler - handles all the incoming authorization headers and validates them if possible.
+type authHandler struct {
+	handler http.Handler
+}
+
+// setAuthHandler to validate authorization header for the incoming request.
+func setAuthHandler(h http.Handler) http.Handler {
+	return authHandler{h}
+}
+
+// List of all support S3 auth types.
+var supportedS3AuthTypes = []authType{
+	authTypeAnonymous,
+	authTypePresigned,
+	authTypeSigned,
+	authTypePostPolicy,
+	authTypeStreamingSigned,
+}
+
+// Validate if the authType is valid and supported.
+func isSupportedS3AuthType(aType authType) bool {
+	for _, a := range supportedS3AuthTypes {
+		if a == aType {
+			return true
+		}
+	}
+	return false
+}
+
+// handler for validating incoming authorization headers.
+func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	aType := getRequestAuthType(r)
+	if isSupportedS3AuthType(aType) {
+		// Let top level caller validate for anonymous and known signed requests.
+		a.handler.ServeHTTP(w, r)
+		return
+	} else if aType == authTypeJWT {
+		// Validate Authorization header if its valid for JWT request.
+		if !isJWTReqAuthenticated(r) {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		a.handler.ServeHTTP(w, r)
+		return
+	}
+	writeErrorResponse(w, r, ErrSignatureVersionNotSupported, r.URL.Path)
+}

cmd/auth-handler_test.go

@@ -0,0 +1,206 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"testing"
+)
+
+// Test all s3 supported auth types.
+func TestS3SupportedAuthType(t *testing.T) {
+	type testCase struct {
+		authT authType
+		pass  bool
+	}
+	// List of all valid and invalid test cases.
+	testCases := []testCase{
+		// Test 1 - supported s3 type anonymous.
+		{
+			authT: authTypeAnonymous,
+			pass:  true,
+		},
+		// Test 2 - supported s3 type presigned.
+		{
+			authT: authTypePresigned,
+			pass:  true,
+		},
+		// Test 3 - supported s3 type signed.
+		{
+			authT: authTypeSigned,
+			pass:  true,
+		},
+		// Test 4 - supported s3 type with post policy.
+		{
+			authT: authTypePostPolicy,
+			pass:  true,
+		},
+		// Test 5 - supported s3 type with streaming signed.
+		{
+			authT: authTypeStreamingSigned,
+			pass:  true,
+		},
+		// Test 6 - JWT is not supported s3 type.
+		{
+			authT: authTypeJWT,
+			pass:  false,
+		},
+		// Test 7 - unknown auth header is not supported s3 type.
+		{
+			authT: authTypeUnknown,
+			pass:  false,
+		},
+		// Test 8 - some new auth type is not supported s3 type.
+		{
+			authT: authType(7),
+			pass:  false,
+		},
+	}
+	// Validate all the test cases.
+	for i, tt := range testCases {
+		ok := isSupportedS3AuthType(tt.authT)
+		if ok != tt.pass {
+			t.Errorf("Test %d:, Expected %t, got %t", i+1, tt.pass, ok)
+		}
+	}
+}
+
+// TestIsRequestUnsignedPayload - Test validates the Unsigned payload detection logic.
+func TestIsRequestUnsignedPayload(t *testing.T) {
+	testCases := []struct {
+		inputAmzContentHeader string
+		expectedResult        bool
+	}{
+		// Test case - 1.
+		// Test case with "X-Amz-Content-Sha256" header set to empty value.
+		{"", false},
+		// Test case - 2.
+		// Test case with "X-Amz-Content-Sha256" header set to  "UNSIGNED-PAYLOAD"
+		// The payload is flagged as unsigned When "X-Amz-Content-Sha256" header is set to  "UNSIGNED-PAYLOAD".
+		{"UNSIGNED-PAYLOAD", true},
+		// Test case - 3.
+		// set to a random value.
+		{"abcd", false},
+	}
+
+	// creating an input HTTP request.
+	// Only the headers are relevant for this particular test.
+	inputReq, err := http.NewRequest("GET", "http://example.com", nil)
+	if err != nil {
+		t.Fatalf("Error initializing input HTTP request: %v", err)
+	}
+
+	for i, testCase := range testCases {
+		inputReq.Header.Set("X-Amz-Content-Sha256", testCase.inputAmzContentHeader)
+		actualResult := isRequestUnsignedPayload(inputReq)
+		if testCase.expectedResult != actualResult {
+			t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult)
+		}
+	}
+}
+
+// TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature verision v4 detection.
+func TestIsRequestPresignedSignatureV4(t *testing.T) {
+	testCases := []struct {
+		inputQueryKey   string
+		inputQueryValue string
+		expectedResult  bool
+	}{
+		// Test case - 1.
+		// Test case with query key ""X-Amz-Credential" set.
+		{"", "", false},
+		// Test case - 2.
+		{"X-Amz-Credential", "", true},
+		// Test case - 3.
+		{"X-Amz-Content-Sha256", "", false},
+	}
+
+	for i, testCase := range testCases {
+		// creating an input HTTP request.
+		// Only the query parameters are relevant for this particular test.
+		inputReq, err := http.NewRequest("GET", "http://example.com", nil)
+		if err != nil {
+			t.Fatalf("Error initializing input HTTP request: %v", err)
+		}
+		q := inputReq.URL.Query()
+		q.Add(testCase.inputQueryKey, testCase.inputQueryValue)
+		inputReq.URL.RawQuery = q.Encode()
+
+		actualResult := isRequestPresignedSignatureV4(inputReq)
+		if testCase.expectedResult != actualResult {
+			t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult)
+		}
+	}
+}
+
+// Provides a fully populated http request instance, fails otherwise.
+func mustNewRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+	req, err := newTestRequest(method, urlStr, contentLength, body)
+	if err != nil {
+		t.Fatalf("Unable to initialize new http request %s", err)
+	}
+	return req
+}
+
+// This is similar to mustNewRequest but additionally the request
+// is signed with AWS Signature V4, fails if not able to do so.
+func mustNewSignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+	req := mustNewRequest(method, urlStr, contentLength, body, t)
+	cred := serverConfig.GetCredential()
+	if err := signRequest(req, cred.AccessKeyID, cred.SecretAccessKey); err != nil {
+		t.Fatalf("Unable to inititalized new signed http request %s", err)
+	}
+	return req
+}
+
+// Tests is requested authenticated function, tests replies for s3 errors.
+func TestIsReqAuthenticated(t *testing.T) {
+	path, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(path)
+
+	serverConfig.SetCredential(credential{"myuser", "mypassword"})
+
+	// List of test cases for validating http request authentication.
+	testCases := []struct {
+		req     *http.Request
+		s3Error APIErrorCode
+	}{
+		// When request is nil, internal error is returned.
+		{nil, ErrInternalError},
+		// When request is unsigned, access denied is returned.
+		{mustNewRequest("GET", "http://localhost:9000", 0, nil, t), ErrAccessDenied},
+		// When request is properly signed, but has bad Content-MD5 header.
+		{mustNewSignedRequest("PUT", "http://localhost:9000", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
+		// When request is properly signed, error is none.
+		{mustNewSignedRequest("GET", "http://localhost:9000", 0, nil, t), ErrNone},
+	}
+
+	// Validates all testcases.
+	for _, testCase := range testCases {
+		if testCase.s3Error == ErrBadDigest {
+			testCase.req.Header.Set("Content-Md5", "garbage")
+		}
+		if s3Error := isReqAuthenticated(testCase.req); s3Error != testCase.s3Error {
+			t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
+		}
+	}
+}

cmd/benchmark-utils_test.go

@@ -0,0 +1,370 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"io/ioutil"
+	"math"
+	"math/rand"
+	"strconv"
+	"testing"
+	"time"
+)
+
+// Benchmark utility functions for ObjectLayer.PutObject().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
+func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// PutObject returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum string
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for PutObject.
+	hasher := md5.New()
+	hasher.Write([]byte(textData))
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// insert the object.
+		md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if md5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// Benchmark utility functions for ObjectLayer.PutObjectPart().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObjectPart benchmark.
+func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	objSize := 128 * 1024 * 1024
+
+	// PutObjectPart returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum, uploadID string
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for NewMultipartUpload.
+	hasher := md5.New()
+	hasher.Write([]byte(textData))
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+	uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	var textPartData []byte
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObjectPart starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// insert the object.
+		totalPartsNR := int(math.Ceil(float64(objSize) / float64(partSize)))
+		for j := 0; j < totalPartsNR; j++ {
+			hasher.Reset()
+			if j < totalPartsNR-1 {
+				textPartData = textData[j*partSize : (j+1)*partSize-1]
+			} else {
+				textPartData = textData[j*partSize:]
+			}
+			hasher.Write([]byte(textPartData))
+			metadata := make(map[string]string)
+			metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+			md5Sum, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"])
+			if err != nil {
+				b.Fatal(err)
+			}
+			if md5Sum != metadata["md5Sum"] {
+				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+			}
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
+func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// uses *testing.B and the object Layer to run the benchmark.
+	runPutObjectPartBenchmark(b, objLayer, objSize)
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
+func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// uses *testing.B and the object Layer to run the benchmark.
+	runPutObjectBenchmark(b, objLayer, objSize)
+}
+
+// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
+func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created on function return.
+	defer removeRoots(disks)
+	// uses *testing.B and the object Layer to run the benchmark.
+	runPutObjectBenchmarkParallel(b, objLayer, objSize)
+}
+
+// Benchmark utility functions for ObjectLayer.GetObject().
+// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
+func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// PutObject returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum string
+	for i := 0; i < 10; i++ {
+		// get text data generated for number of bytes equal to object size.
+		textData := generateBytesData(objSize)
+		// generate md5sum for the generated data.
+		// md5sum of the data to written is required as input for PutObject.
+		// PutObject is the functions which writes the data onto the FS/XL backend.
+		hasher := md5.New()
+		hasher.Write([]byte(textData))
+		metadata := make(map[string]string)
+		metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+		// insert the object.
+		md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if md5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+		}
+	}
+
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for GetObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		var buffer = new(bytes.Buffer)
+		err = obj.GetObject(bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer)
+		if err != nil {
+			b.Error(err)
+		}
+	}
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+
+}
+
+// randomly picks a character and returns its equivalent byte array.
+func getRandomByte() []byte {
+	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	// seeding the random number generator.
+	rand.Seed(time.Now().UnixNano())
+	var b byte
+	// pick a character randomly.
+	b = letterBytes[rand.Intn(len(letterBytes))]
+	return []byte{b}
+}
+
+// picks a random byte and repeats it to size bytes.
+func generateBytesData(size int) []byte {
+	// repeat the random character chosen size
+	return bytes.Repeat(getRandomByte(), size)
+}
+
+// creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
+func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created.
+	defer removeRoots(disks)
+	//  uses *testing.B and the object Layer to run the benchmark.
+	runGetObjectBenchmark(b, objLayer, objSize)
+}
+
+// creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
+func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
+	// create a temp XL/FS backend.
+	objLayer, disks, err := makeTestBackend(instanceType)
+	if err != nil {
+		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
+	}
+	// cleaning up the backend by removing all the directories and files created.
+	defer removeRoots(disks)
+	//  uses *testing.B and the object Layer to run the benchmark.
+	runGetObjectBenchmarkParallel(b, objLayer, objSize)
+}
+
+// Parallel benchmark utility functions for ObjectLayer.PutObject().
+// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
+func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// PutObject returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum string
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for PutObject.
+	hasher := md5.New()
+	hasher.Write([]byte(textData))
+	metadata := make(map[string]string)
+	metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+
+	b.RunParallel(func(pb *testing.PB) {
+		i := 0
+		for pb.Next() {
+			// insert the object.
+			md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
+			if err != nil {
+				b.Fatal(err)
+			}
+			if md5Sum != metadata["md5Sum"] {
+				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", md5Sum, metadata["md5Sum"])
+			}
+			i++
+		}
+	})
+
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+}
+
+// Parallel benchmark utility functions for ObjectLayer.GetObject().
+// Creates Object layer setup ( MakeBucket, PutObject) and then runs the benchmark.
+func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
+	var err error
+	// obtains random bucket name.
+	bucket := getRandomBucketName()
+	// create bucket.
+	err = obj.MakeBucket(bucket)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// PutObject returns md5Sum of the object inserted.
+	// md5Sum variable is assigned with that value.
+	var md5Sum string
+	for i := 0; i < 10; i++ {
+		// get text data generated for number of bytes equal to object size.
+		textData := generateBytesData(objSize)
+		// generate md5sum for the generated data.
+		// md5sum of the data to written is required as input for PutObject.
+		// PutObject is the functions which writes the data onto the FS/XL backend.
+		hasher := md5.New()
+		hasher.Write([]byte(textData))
+		metadata := make(map[string]string)
+		metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+		// insert the object.
+		md5Sum, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if md5Sum != metadata["md5Sum"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+		}
+	}
+
+	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
+	b.ReportAllocs()
+	// the actual benchmark for GetObject starts here. Reset the benchmark timer.
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		i := 0
+		for pb.Next() {
+			err = obj.GetObject(bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard)
+			if err != nil {
+				b.Error(err)
+			}
+			i++
+			if i == 10 {
+				i = 0
+			}
+		}
+	})
+	// Benchmark ends here. Stop timer.
+	b.StopTimer()
+
+}

cmd/bucket-handlers-listobjects.go

@@ -0,0 +1,167 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/gorilla/mux"
+)
+
+// Validate all the ListObjects query arguments, returns an APIErrorCode
+// if one of the args do not meet the required conditions.
+// Special conditions required by Minio server are as below
+// - delimiter if set should be equal to '/', otherwise the request is rejected.
+// - marker if set should have a common prefix with 'prefix' param, otherwise
+//   the request is rejected.
+func listObjectsValidateArgs(prefix, marker, delimiter string, maxKeys int) APIErrorCode {
+	// Max keys cannot be negative.
+	if maxKeys < 0 {
+		return ErrInvalidMaxKeys
+	}
+
+	/// Minio special conditions for ListObjects.
+
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		return ErrNotImplemented
+	}
+	// Marker is set validate pre-condition.
+	if marker != "" {
+		// Marker not common with prefix is not implemented.
+		if !strings.HasPrefix(marker, prefix) {
+			return ErrNotImplemented
+		}
+	}
+	// Success.
+	return ErrNone
+}
+
+// ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
+// --------------------------
+// This implementation of the GET operation returns some or all (up to 1000)
+// of the objects in a bucket. You can use the request parameters as selection
+// criteria to return a subset of the objects in a bucket.
+//
+// NOTE: It is recommended that this API to be used for application development.
+// Minio continues to support ListObjectsV1 for supporting legacy tools.
+func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:ListBucket", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypeSigned, authTypePresigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+	// Extract all the listObjectsV2 query params to their native values.
+	prefix, token, startAfter, delimiter, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+
+	// In ListObjectsV2 'continuation-token' is the marker.
+	marker := token
+	// Check if 'continuation-token' is empty.
+	if token == "" {
+		// Then we need to use 'start-after' as marker instead.
+		marker = startAfter
+	}
+	// Validate all the query params before beginning to serve the request.
+	if s3Error := listObjectsValidateArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+	// Inititate a list objects operation based on the input params.
+	// On success would return back ListObjectsInfo object to be
+	// marshalled into S3 compatible XML header.
+	listObjectsInfo, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
+	if err != nil {
+		errorIf(err, "Unable to list objects.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, maxKeys, listObjectsInfo)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodeResponse(response))
+}
+
+// ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
+// --------------------------
+// This implementation of the GET operation returns some or all (up to 1000)
+// of the objects in a bucket. You can use the request parameters as selection
+// criteria to return a subset of the objects in a bucket.
+//
+func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:ListBucket", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypeSigned, authTypePresigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// Extract all the litsObjectsV1 query params to their native values.
+	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
+
+	// Validate all the query params before beginning to serve the request.
+	if s3Error := listObjectsValidateArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Inititate a list objects operation based on the input params.
+	// On success would return back ListObjectsInfo object to be
+	// marshalled into S3 compatible XML header.
+	listObjectsInfo, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
+	if err != nil {
+		errorIf(err, "Unable to list objects.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodeResponse(response))
+}

cmd/bucket-handlers.go

@@ -0,0 +1,447 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	mux "github.com/gorilla/mux"
+)
+
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+// Enforces bucket policies for a bucket for a given tatusaction.
+func enforceBucketPolicy(bucket string, action string, reqURL *url.URL) (s3Error APIErrorCode) {
+	if !IsValidBucketName(bucket) {
+		return ErrInvalidBucketName
+	}
+	// Fetch bucket policy, if policy is not set return access denied.
+	policy := globalBucketPolicies.GetBucketPolicy(bucket)
+	if policy == nil {
+		return ErrAccessDenied
+	}
+
+	// Construct resource in 'arn:aws:s3:::examplebucket/object' format.
+	resource := AWSResourcePrefix + strings.TrimPrefix(reqURL.Path, "/")
+
+	// Get conditions for policy verification.
+	conditions := make(map[string]string)
+	for queryParam := range reqURL.Query() {
+		conditions[queryParam] = reqURL.Query().Get(queryParam)
+	}
+
+	// Validate action, resource and conditions with current policy statements.
+	if !bucketPolicyEvalStatements(action, resource, conditions, policy.Statements) {
+		return ErrAccessDenied
+	}
+	return ErrNone
+}
+
+// GetBucketLocationHandler - GET Bucket location.
+// -------------------------
+// This operation returns bucket location.
+func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:GetBucketLocation", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypeSigned, authTypePresigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	if _, err := api.ObjectAPI.GetBucketInfo(bucket); err != nil {
+		errorIf(err, "Unable to fetch bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Generate response.
+	encodedSuccessResponse := encodeResponse(LocationResponse{})
+	// Get current region.
+	region := serverConfig.GetRegion()
+	if region != "us-east-1" {
+		encodedSuccessResponse = encodeResponse(LocationResponse{
+			Location: region,
+		})
+	}
+	setCommonHeaders(w) // Write headers.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// ListMultipartUploadsHandler - GET Bucket (List Multipart uploads)
+// -------------------------
+// This operation lists in-progress multipart uploads. An in-progress
+// multipart upload is a multipart upload that has been initiated,
+// using the Initiate Multipart Upload request, but has not yet been
+// completed or aborted. This operation returns at most 1,000 multipart
+// uploads in the response.
+//
+func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:ListBucketMultipartUploads", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
+	if maxUploads < 0 {
+		writeErrorResponse(w, r, ErrInvalidMaxUploads, r.URL.Path)
+		return
+	}
+	if keyMarker != "" {
+		// Marker not common with prefix is not implemented.
+		if !strings.HasPrefix(keyMarker, prefix) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
+	}
+
+	listMultipartsInfo, err := api.ObjectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
+	if err != nil {
+		errorIf(err, "Unable to list multipart uploads.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// generate response
+	response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo)
+	encodedSuccessResponse := encodeResponse(response)
+	// write headers.
+	setCommonHeaders(w)
+	// write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// ListBucketsHandler - GET Service
+// -----------
+// This implementation of the GET operation returns a list of all buckets
+// owned by the authenticated sender of the request.
+func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
+	// List buckets does not support bucket policies, no need to enforce it.
+	if s3Error := checkAuth(r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	bucketsInfo, err := api.ObjectAPI.ListBuckets()
+	if err != nil {
+		errorIf(err, "Unable to list buckets.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Generate response.
+	response := generateListBucketsResponse(bucketsInfo)
+	encodedSuccessResponse := encodeResponse(response)
+	// Write headers.
+	setCommonHeaders(w)
+	// Write response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// DeleteMultipleObjectsHandler - deletes multiple objects.
+func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:DeleteObject", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// Content-Length is required and should be non-zero
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if r.ContentLength <= 0 {
+		writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+		return
+	}
+
+	// Content-Md5 is requied should be set
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if _, ok := r.Header["Content-Md5"]; !ok {
+		writeErrorResponse(w, r, ErrMissingContentMD5, r.URL.Path)
+		return
+	}
+
+	// Allocate incoming content length bytes.
+	deleteXMLBytes := make([]byte, r.ContentLength)
+
+	// Read incoming body XML bytes.
+	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
+		errorIf(err, "Unable to read HTTP body.")
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
+
+	// Unmarshal list of keys to be deleted.
+	deleteObjects := &DeleteObjectsRequest{}
+	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
+		errorIf(err, "Unable to unmarshal delete objects request XML.")
+		writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
+		return
+	}
+
+	var deleteErrors []DeleteError
+	var deletedObjects []ObjectIdentifier
+	// Loop through all the objects and delete them sequentially.
+	for _, object := range deleteObjects.Objects {
+		err := api.ObjectAPI.DeleteObject(bucket, object.ObjectName)
+		if err == nil {
+			deletedObjects = append(deletedObjects, ObjectIdentifier{
+				ObjectName: object.ObjectName,
+			})
+		} else {
+			errorIf(err, "Unable to delete object.")
+			deleteErrors = append(deleteErrors, DeleteError{
+				Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
+				Message: errorCodeResponse[toAPIErrorCode(err)].Description,
+				Key:     object.ObjectName,
+			})
+		}
+	}
+	// Generate response
+	response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
+	encodedSuccessResponse := encodeResponse(response)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// PutBucketHandler - PUT Bucket
+// ----------
+// This implementation of the PUT operation creates a new bucket for authenticated request
+func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
+	// PutBucket does not support policies, use checkAuth to validate signature.
+	if s3Error := checkAuth(r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Validate if incoming location constraint is valid, reject
+	// requests which do not follow valid region requirements.
+	if s3Error := isValidLocationConstraint(r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+	}
+
+	// Proceed to creating a bucket.
+	err := api.ObjectAPI.MakeBucket(bucket)
+	if err != nil {
+		errorIf(err, "Unable to create a bucket.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// Make sure to add Location information here only for bucket
+	w.Header().Set("Location", getLocation(r))
+	writeSuccessResponse(w, nil)
+}
+
+// PostPolicyBucketHandler - POST policy
+// ----------
+// This implementation of the POST operation handles object creation with a specified
+// signature policy in multipart/form-data
+func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
+	// Here the parameter is the size of the form data that should
+	// be loaded in memory, the remaining being put in temporary files.
+	reader, err := r.MultipartReader()
+	if err != nil {
+		errorIf(err, "Unable to initialize multipart reader.")
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
+		return
+	}
+
+	fileBody, fileName, formValues, err := extractPostPolicyFormValues(reader)
+	if err != nil {
+		errorIf(err, "Unable to parse form values.")
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
+		return
+	}
+	bucket := mux.Vars(r)["bucket"]
+	formValues["Bucket"] = bucket
+	object := formValues["Key"]
+
+	if fileName != "" && strings.Contains(object, "${filename}") {
+		// S3 feature to replace ${filename} found in Key form field
+		// by the filename attribute passed in multipart
+		object = strings.Replace(object, "${filename}", fileName, -1)
+	}
+
+	// Verify policy signature.
+	apiErr := doesPolicySignatureMatch(formValues)
+	if apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+	if apiErr = checkPostPolicy(formValues); apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+
+	// Save metadata.
+	metadata := make(map[string]string)
+	// Nothing to store right now.
+
+	md5Sum, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, metadata)
+	if err != nil {
+		errorIf(err, "Unable to create object.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	if md5Sum != "" {
+		w.Header().Set("ETag", "\""+md5Sum+"\"")
+	}
+	encodedSuccessResponse := encodeResponse(PostResponse{
+		Location: getObjectLocation(bucket, object), // TODO Full URL is preferred
+		Bucket:   bucket,
+		Key:      object,
+		ETag:     md5Sum,
+	})
+
+	setCommonHeaders(w)
+
+	writeSuccessResponse(w, encodedSuccessResponse)
+
+	// Fetch object info for notifications.
+	objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
+	if err != nil {
+		errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
+		return
+	}
+
+	if eventN.IsBucketNotificationSet(bucket) {
+		// Notify object created event.
+		eventNotify(eventData{
+			Type:    ObjectCreatedPost,
+			Bucket:  bucket,
+			ObjInfo: objInfo,
+			ReqParams: map[string]string{
+				"sourceIPAddress": r.RemoteAddr,
+			},
+		})
+	}
+}
+
+// HeadBucketHandler - HEAD Bucket
+// ----------
+// This operation is useful to determine if a bucket exists.
+// The operation returns a 200 OK if the bucket exists and you
+// have permission to access it. Otherwise, the operation might
+// return responses such as 404 Not Found and 403 Forbidden.
+func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:ListBucket", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	if _, err := api.ObjectAPI.GetBucketInfo(bucket); err != nil {
+		errorIf(err, "Unable to fetch bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// DeleteBucketHandler - Delete bucket
+func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
+	// DeleteBucket does not support bucket policies, use checkAuth to validate signature.
+	if s3Error := checkAuth(r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Attempt to delete bucket.
+	if err := api.ObjectAPI.DeleteBucket(bucket); err != nil {
+		errorIf(err, "Unable to delete a bucket.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Delete bucket access policy, if present - ignore any errors.
+	removeBucketPolicy(bucket, api.ObjectAPI)
+
+	// Delete notification config, if present - ignore any errors.
+	removeNotificationConfig(bucket, api.ObjectAPI)
+
+	// Write success response.
+	writeSuccessNoContent(w)
+}

cmd/bucket-notification-datatypes.go

@@ -0,0 +1,178 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"errors"
+)
+
+// Represents the criteria for the filter rule.
+type filterRule struct {
+	Name  string `xml:"Name"`
+	Value string `xml:"Value"`
+}
+
+// Collection of filter rules per service config.
+type keyFilter struct {
+	FilterRules []filterRule `xml:"FilterRule,omitempty"`
+}
+
+// Queue SQS configuration.
+type queueConfig struct {
+	Events []string `xml:"Event"`
+	Filter struct {
+		Key keyFilter `xml:"S3Key,omitempty"`
+	}
+	ID       string `xml:"Id"`
+	QueueARN string `xml:"Queue"`
+}
+
+// Topic SNS configuration, this is a compliance field not used by minio yet.
+type topicConfig struct {
+	Events []string `xml:"Event"`
+	Filter struct {
+		Key keyFilter `xml:"S3Key"`
+	}
+	ID       string `xml:"Id"`
+	TopicARN string `xml:"Topic"`
+}
+
+// Lambda function configuration, this is a compliance field not used by minio yet.
+type lambdaConfig struct {
+	Events []string `xml:"Event"`
+	Filter struct {
+		Key keyFilter `xml:"S3Key,omitempty"`
+	}
+	ID        string `xml:"Id"`
+	LambdaARN string `xml:"CloudFunction"`
+}
+
+// Notification configuration structure represents the XML format of
+// notification configuration of buckets.
+type notificationConfig struct {
+	XMLName       xml.Name       `xml:"NotificationConfiguration"`
+	QueueConfigs  []queueConfig  `xml:"QueueConfiguration"`
+	TopicConfigs  []topicConfig  `xml:"TopicConfiguration"`
+	LambdaConfigs []lambdaConfig `xml:"CloudFunctionConfiguration"`
+}
+
+// Internal error used to signal notifications not set.
+var errNoSuchNotifications = errors.New("The specified bucket does not have bucket notifications")
+
+// EventName is AWS S3 event type:
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html
+type EventName int
+
+const (
+	// ObjectCreatedPut is s3:ObjectCreated:Put
+	ObjectCreatedPut EventName = iota
+	// ObjectCreatedPost is s3:ObjectCreated:Post
+	ObjectCreatedPost
+	// ObjectCreatedCopy is s3:ObjectCreated:Copy
+	ObjectCreatedCopy
+	// ObjectCreatedCompleteMultipartUpload is s3:ObjectCreated:CompleteMultipartUpload
+	ObjectCreatedCompleteMultipartUpload
+	// ObjectRemovedDelete is s3:ObjectRemoved:Delete
+	ObjectRemovedDelete
+)
+
+// Stringer interface for event name.
+func (eventName EventName) String() string {
+	switch eventName {
+	case ObjectCreatedPut:
+		return "s3:ObjectCreated:Put"
+	case ObjectCreatedPost:
+		return "s3:ObjectCreated:Post"
+	case ObjectCreatedCopy:
+		return "s3:ObjectCreated:Copy"
+	case ObjectCreatedCompleteMultipartUpload:
+		return "s3:ObjectCreated:CompleteMultipartUpload"
+	case ObjectRemovedDelete:
+		return "s3:ObjectRemoved:Delete"
+	default:
+		return "s3:Unknown"
+	}
+}
+
+// Indentity represents the user id, this is a compliance field.
+type identity struct {
+	PrincipalID string `json:"principalId"`
+}
+
+func defaultIdentity() identity {
+	return identity{"minio"}
+}
+
+// Notification event bucket metadata.
+type bucketMeta struct {
+	Name          string   `json:"name"`
+	OwnerIdentity identity `json:"ownerIdentity"`
+	ARN           string   `json:"arn"`
+}
+
+// Notification event object metadata.
+type objectMeta struct {
+	Key       string `json:"key"`
+	Size      int64  `json:"size,omitempty"`
+	ETag      string `json:"eTag,omitempty"`
+	VersionID string `json:"versionId,omitempty"`
+	Sequencer string `json:"sequencer"`
+}
+
+// Notification event server specific metadata.
+type eventMeta struct {
+	SchemaVersion   string     `json:"s3SchemaVersion"`
+	ConfigurationID string     `json:"configurationId"`
+	Bucket          bucketMeta `json:"bucket"`
+	Object          objectMeta `json:"object"`
+}
+
+// NotificationEvent represents an Amazon an S3 bucket notification event.
+type NotificationEvent struct {
+	EventVersion      string            `json:"eventVersion"`
+	EventSource       string            `json:"eventSource"`
+	AwsRegion         string            `json:"awsRegion"`
+	EventTime         string            `json:"eventTime"`
+	EventName         string            `json:"eventName"`
+	UserIdentity      identity          `json:"userIdentity"`
+	RequestParameters map[string]string `json:"requestParameters"`
+	ResponseElements  map[string]string `json:"responseElements"`
+	S3                eventMeta         `json:"s3"`
+}
+
+// Represents the minio topic type and account id's.
+type arnTopic struct {
+	Type      string
+	AccountID string
+}
+
+// Stringer for constructing AWS ARN compatible string.
+func (m arnTopic) String() string {
+	return minioTopic + serverConfig.GetRegion() + ":" + m.AccountID + ":" + m.Type
+}
+
+// Represents the minio sqs type and account id's.
+type arnSQS struct {
+	Type      string
+	AccountID string
+}
+
+// Stringer for constructing AWS ARN compatible string.
+func (m arnSQS) String() string {
+	return minioSqs + serverConfig.GetRegion() + ":" + m.AccountID + ":" + m.Type
+}

cmd/bucket-notification-handlers.go

@@ -0,0 +1,269 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"io"
+	"net/http"
+	"path"
+	"time"
+
+	"github.com/gorilla/mux"
+)
+
+const (
+	bucketConfigPrefix       = "buckets"
+	bucketNotificationConfig = "notification.xml"
+)
+
+// GetBucketNotificationHandler - This implementation of the GET
+// operation uses the notification subresource to return the
+// notification configuration of a bucket. If notifications are
+// not enabled on the bucket, the operation returns an empty
+// NotificationConfiguration element.
+func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
+	// Validate request authorization.
+	if s3Error := checkAuth(r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	// Attempt to successfully load notification config.
+	nConfig, err := loadNotificationConfig(bucket, api.ObjectAPI)
+	if err != nil && err != errNoSuchNotifications {
+		errorIf(err, "Unable to read notification configuration.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// For no notifications we write a dummy XML.
+	if err == errNoSuchNotifications {
+		// Complies with the s3 behavior in this regard.
+		nConfig = &notificationConfig{}
+	}
+	notificationBytes, err := xml.Marshal(nConfig)
+	if err != nil {
+		// For any marshalling failure.
+		errorIf(err, "Unable to marshal notification configuration into XML.", err)
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// Success.
+	writeSuccessResponse(w, notificationBytes)
+}
+
+// PutBucketNotificationHandler - Minio notification feature enables
+// you to receive notifications when certain events happen in your bucket.
+// Using this API, you can replace an existing notification configuration.
+// The configuration is an XML file that defines the event types that you
+// want Minio to publish and the destination where you want Minio to publish
+// an event notification when it detects an event of the specified type.
+// By default, your bucket has no event notifications configured. That is,
+// the notification configuration will be an empty NotificationConfiguration.
+func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
+	// Validate request authorization.
+	if s3Error := checkAuth(r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	_, err := api.ObjectAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err, "Unable to bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// If Content-Length is unknown or zero, deny the request. PutBucketNotification
+	// always needs a Content-Length if incoming request is not chunked.
+	if !contains(r.TransferEncoding, "chunked") {
+		if r.ContentLength == -1 {
+			writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+			return
+		}
+	}
+
+	// Reads the incoming notification configuration.
+	var buffer bytes.Buffer
+	var bufferSize int64
+	if r.ContentLength >= 0 {
+		bufferSize, err = io.CopyN(&buffer, r.Body, r.ContentLength)
+	} else {
+		bufferSize, err = io.Copy(&buffer, r.Body)
+	}
+	if err != nil {
+		errorIf(err, "Unable to read incoming body.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	var notificationCfg notificationConfig
+	// Unmarshal notification bytes.
+	notificationConfigBytes := buffer.Bytes()
+	if err = xml.Unmarshal(notificationConfigBytes, &notificationCfg); err != nil {
+		errorIf(err, "Unable to parse notification configuration XML.")
+		writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
+		return
+	} // Successfully marshalled notification configuration.
+
+	// Validate unmarshalled bucket notification configuration.
+	if s3Error := validateNotificationConfig(notificationCfg); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Proceed to save notification configuration.
+	notificationConfigPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
+	_, err = api.ObjectAPI.PutObject(minioMetaBucket, notificationConfigPath, bufferSize, bytes.NewReader(buffer.Bytes()), nil)
+	if err != nil {
+		errorIf(err, "Unable to write bucket notification configuration.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	// Set bucket notification config.
+	eventN.SetBucketNotificationConfig(bucket, &notificationCfg)
+
+	// Success.
+	writeSuccessResponse(w, nil)
+}
+
+// writeNotification marshals notification message before writing to client.
+func writeNotification(w http.ResponseWriter, notification map[string][]NotificationEvent) error {
+	// Invalid response writer.
+	if w == nil {
+		return errInvalidArgument
+	}
+	// Invalid notification input.
+	if notification == nil {
+		return errInvalidArgument
+	}
+	// Marshal notification data into XML and write to client.
+	notificationBytes, err := json.Marshal(&notification)
+	if err != nil {
+		return err
+	}
+	// Add additional CRLF characters for client to
+	// differentiate the individual events properly.
+	_, err = w.Write(append(notificationBytes, crlf...))
+	// Make sure we have flushed, this would set Transfer-Encoding: chunked.
+	w.(http.Flusher).Flush()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// CRLF character used for chunked transfer in accordance with HTTP standards.
+var crlf = []byte("\r\n")
+
+// sendBucketNotification - writes notification back to client on the response writer
+// for each notification input, otherwise writes whitespace characters periodically
+// to keep the connection active. Each notification messages are terminated by CRLF
+// character. Upon any error received on response writer the for loop exits.
+//
+// TODO - do not log for all errors.
+func sendBucketNotification(w http.ResponseWriter, arnListenerCh <-chan []NotificationEvent) {
+	var dummyEvents = map[string][]NotificationEvent{"Records": nil}
+	// Continuously write to client either timely empty structures
+	// every 5 seconds, or return back the notifications.
+	for {
+		select {
+		case events := <-arnListenerCh:
+			if err := writeNotification(w, map[string][]NotificationEvent{"Records": events}); err != nil {
+				errorIf(err, "Unable to write notification to client.")
+				return
+			}
+		case <-time.After(5 * time.Second):
+			if err := writeNotification(w, dummyEvents); err != nil {
+				errorIf(err, "Unable to write notification to client.")
+				return
+			}
+		}
+	}
+}
+
+// ListenBucketNotificationHandler - list bucket notifications.
+func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
+	// Validate request authorization.
+	if s3Error := checkAuth(r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Get notification ARN.
+	topicARN := r.URL.Query().Get("notificationARN")
+	if topicARN == "" {
+		writeErrorResponse(w, r, ErrARNNotification, r.URL.Path)
+		return
+	}
+
+	// Validate if bucket exists.
+	_, err := api.ObjectAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err, "Unable to bucket info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	notificationCfg := eventN.GetBucketNotificationConfig(bucket)
+	if notificationCfg == nil {
+		writeErrorResponse(w, r, ErrARNNotification, r.URL.Path)
+		return
+	}
+
+	// Set SNS notifications only if special "listen" sns is set in bucket
+	// notification configs.
+	if !isMinioSNSConfigured(topicARN, notificationCfg.TopicConfigs) {
+		writeErrorResponse(w, r, ErrARNNotification, r.URL.Path)
+		return
+	}
+
+	// Add all common headers.
+	setCommonHeaders(w)
+
+	// Create a new notification event channel.
+	nEventCh := make(chan []NotificationEvent)
+	// Close the listener channel.
+	defer close(nEventCh)
+
+	// Set sns target.
+	eventN.SetSNSTarget(topicARN, nEventCh)
+	// Remove sns listener after the writer has closed or the client disconnected.
+	defer eventN.RemoveSNSTarget(topicARN, nEventCh)
+
+	// Start sending bucket notifications.
+	sendBucketNotification(w, nEventCh)
+}
+
+// Removes notification.xml for a given bucket, only used during DeleteBucket.
+func removeNotificationConfig(bucket string, objAPI ObjectLayer) error {
+	// Verify bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+
+	notificationConfigPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
+	return objAPI.DeleteObject(minioMetaBucket, notificationConfigPath)
+}

cmd/bucket-notification-handlers_test.go

@@ -0,0 +1,90 @@
+package cmd
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"testing"
+)
+
+// Implement a dummy flush writer.
+type flushWriter struct {
+	io.Writer
+}
+
+// Flush writer is a dummy writer compatible with http.Flusher and http.ResponseWriter.
+func (f *flushWriter) Flush()                            {}
+func (f *flushWriter) Write(b []byte) (n int, err error) { return f.Writer.Write(b) }
+func (f *flushWriter) Header() http.Header               { return http.Header{} }
+func (f *flushWriter) WriteHeader(code int)              {}
+
+func newFlushWriter(writer io.Writer) *flushWriter {
+	return &flushWriter{writer}
+}
+
+// Tests write notification code.
+func TestWriteNotification(t *testing.T) {
+	// Initialize a new test config.
+	root, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Unable to initialize test config %s", err)
+	}
+	defer removeAll(root)
+
+	var buffer bytes.Buffer
+	// Collection of test cases for each event writer.
+	testCases := []struct {
+		writer *flushWriter
+		event  map[string][]NotificationEvent
+		err    error
+	}{
+		// Invalid input argument with writer `nil` - Test - 1
+		{
+			writer: nil,
+			event:  nil,
+			err:    errInvalidArgument,
+		},
+		// Invalid input argument with event `nil` - Test - 2
+		{
+			writer: newFlushWriter(ioutil.Discard),
+			event:  nil,
+			err:    errInvalidArgument,
+		},
+		// Unmarshal and write, validate last 5 bytes. - Test - 3
+		{
+			writer: newFlushWriter(&buffer),
+			event: map[string][]NotificationEvent{
+				"Records": {newNotificationEvent(eventData{
+					Type:   ObjectCreatedPut,
+					Bucket: "testbucket",
+					ObjInfo: ObjectInfo{
+						Name: "key",
+					},
+					ReqParams: map[string]string{
+						"ip": "10.1.10.1",
+					}}),
+				},
+			},
+			err: nil,
+		},
+	}
+	// Validates all the testcases for writing notification.
+	for _, testCase := range testCases {
+		err := writeNotification(testCase.writer, testCase.event)
+		if err != testCase.err {
+			t.Errorf("Unable to write notification %s", err)
+		}
+		// Validates if the ending string has 'crlf'
+		if err == nil && !bytes.HasSuffix(buffer.Bytes(), crlf) {
+			buf := buffer.Bytes()[buffer.Len()-5 : 0]
+			t.Errorf("Invalid suffix found from the writer last 5 bytes %s, expected `\r\n`", string(buf))
+		}
+		// Not printing 'buf' on purpose, validates look for string '10.1.10.1'.
+		if err == nil && !bytes.Contains(buffer.Bytes(), []byte("10.1.10.1")) {
+			// Enable when debugging)
+			// fmt.Println(string(buffer.Bytes()))
+			t.Errorf("Requested content couldn't be found, expected `10.1.10.1`")
+		}
+	}
+}

cmd/bucket-notification-utils.go

@@ -0,0 +1,322 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "strings"
+
+// List of valid event types.
+var suppportedEventTypes = map[string]struct{}{
+	// Object created event types.
+	"s3:ObjectCreated:*":                       {},
+	"s3:ObjectCreated:Put":                     {},
+	"s3:ObjectCreated:Post":                    {},
+	"s3:ObjectCreated:Copy":                    {},
+	"s3:ObjectCreated:CompleteMultipartUpload": {},
+	// Object removed event types.
+	"s3:ObjectRemoved:*":      {},
+	"s3:ObjectRemoved:Delete": {},
+}
+
+// checkEvent - checks if an event is supported.
+func checkEvent(event string) APIErrorCode {
+	_, ok := suppportedEventTypes[event]
+	if !ok {
+		return ErrEventNotification
+	}
+	return ErrNone
+}
+
+// checkEvents - checks given list of events if all of them are valid.
+// given if one of them is invalid, this function returns an error.
+func checkEvents(events []string) APIErrorCode {
+	for _, event := range events {
+		if s3Error := checkEvent(event); s3Error != ErrNone {
+			return s3Error
+		}
+	}
+	return ErrNone
+}
+
+// Valid if filterName is 'prefix'.
+func isValidFilterNamePrefix(filterName string) bool {
+	return "prefix" == filterName
+}
+
+// Valid if filterName is 'suffix'.
+func isValidFilterNameSuffix(filterName string) bool {
+	return "suffix" == filterName
+}
+
+// Is this a valid filterName? - returns true if valid.
+func isValidFilterName(filterName string) bool {
+	return isValidFilterNamePrefix(filterName) || isValidFilterNameSuffix(filterName)
+}
+
+// checkFilterRules - checks given list of filter rules if all of them are valid.
+func checkFilterRules(filterRules []filterRule) APIErrorCode {
+	ruleSetMap := make(map[string]string)
+	// Validate all filter rules.
+	for _, filterRule := range filterRules {
+		// Unknown filter rule name found, returns an appropriate error.
+		if !isValidFilterName(filterRule.Name) {
+			return ErrFilterNameInvalid
+		}
+
+		// Filter names should not be set twice per notification service
+		// configuration, if found return an appropriate error.
+		if _, ok := ruleSetMap[filterRule.Name]; ok {
+			if isValidFilterNamePrefix(filterRule.Name) {
+				return ErrFilterNamePrefix
+			} else if isValidFilterNameSuffix(filterRule.Name) {
+				return ErrFilterNameSuffix
+			} else {
+				return ErrFilterNameInvalid
+			}
+		}
+
+		if !IsValidObjectPrefix(filterRule.Value) {
+			return ErrFilterValueInvalid
+		}
+
+		// Set the new rule name to keep track of duplicates.
+		ruleSetMap[filterRule.Name] = filterRule.Value
+	}
+	// Success all prefixes validated.
+	return ErrNone
+}
+
+// Checks validity of input ARN for a given arnType.
+func checkARN(arn, arnType string) APIErrorCode {
+	if !strings.HasPrefix(arn, arnType) {
+		return ErrARNNotification
+	}
+	if !strings.HasPrefix(arn, arnType+serverConfig.GetRegion()+":") {
+		return ErrRegionNotification
+	}
+	account := strings.SplitN(strings.TrimPrefix(arn, arnType+serverConfig.GetRegion()+":"), ":", 2)
+	switch len(account) {
+	case 1:
+		// This means ARN is malformed, account should have min of 2elements.
+		return ErrARNNotification
+	case 2:
+		// Account topic id or topic name cannot be empty.
+		if account[0] == "" || account[1] == "" {
+			return ErrARNNotification
+		}
+	}
+	return ErrNone
+}
+
+// checkQueueARN - check if the queue arn is valid.
+func checkQueueARN(queueARN string) APIErrorCode {
+	return checkARN(queueARN, minioSqs)
+}
+
+// checkTopicARN - check if the topic arn is valid.
+func checkTopicARN(topicARN string) APIErrorCode {
+	return checkARN(topicARN, minioTopic)
+}
+
+// Returns true if the topicARN is for an Minio sns listen type.
+func isMinioSNS(topicARN arnTopic) bool {
+	return strings.HasSuffix(topicARN.Type, snsTypeMinio)
+}
+
+// isMinioSNSConfigured - verifies if one topic ARN is valid and is enabled.
+func isMinioSNSConfigured(topicARN string, topicConfigs []topicConfig) bool {
+	for _, topicConfig := range topicConfigs {
+		// Validate if topic ARN is already enabled.
+		if topicARN == topicConfig.TopicARN {
+			return true
+		}
+	}
+	return false
+}
+
+// Validate if we recognize the queue type.
+func isValidQueue(sqsARN arnSQS) bool {
+	amqpQ := isAMQPQueue(sqsARN)       // Is amqp queue?.
+	elasticQ := isElasticQueue(sqsARN) // Is elastic queue?.
+	redisQ := isRedisQueue(sqsARN)     // Is redis queue?.
+	return amqpQ || elasticQ || redisQ
+}
+
+// Validate if we recognize the topic type.
+func isValidTopic(topicARN arnTopic) bool {
+	return isMinioSNS(topicARN) // Is minio topic?.
+}
+
+// Validates account id for input queue ARN.
+func isValidQueueID(queueARN string) bool {
+	// Unmarshals QueueARN into structured object.
+	sqsARN := unmarshalSqsARN(queueARN)
+	// AMQP queue.
+	if isAMQPQueue(sqsARN) {
+		amqpN := serverConfig.GetAMQPNotifyByID(sqsARN.AccountID)
+		return amqpN.Enable && amqpN.URL != ""
+	} else if isElasticQueue(sqsARN) { // Elastic queue.
+		elasticN := serverConfig.GetElasticSearchNotifyByID(sqsARN.AccountID)
+		return elasticN.Enable && elasticN.URL != ""
+
+	} else if isRedisQueue(sqsARN) { // Redis queue.
+		redisN := serverConfig.GetRedisNotifyByID(sqsARN.AccountID)
+		return redisN.Enable && redisN.Addr != ""
+	}
+	return false
+}
+
+// Check - validates queue configuration and returns error if any.
+func checkQueueConfig(qConfig queueConfig) APIErrorCode {
+	// Check queue arn is valid.
+	if s3Error := checkQueueARN(qConfig.QueueARN); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Unmarshals QueueARN into structured object.
+	sqsARN := unmarshalSqsARN(qConfig.QueueARN)
+	// Validate if sqsARN requested any of the known supported queues.
+	if !isValidQueue(sqsARN) {
+		return ErrARNNotification
+	}
+
+	// Validate if the account ID is correct.
+	if !isValidQueueID(qConfig.QueueARN) {
+		return ErrARNNotification
+	}
+
+	// Check if valid events are set in queue config.
+	if s3Error := checkEvents(qConfig.Events); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Check if valid filters are set in queue config.
+	if s3Error := checkFilterRules(qConfig.Filter.Key.FilterRules); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Success.
+	return ErrNone
+}
+
+// Check - validates queue configuration and returns error if any.
+func checkTopicConfig(tConfig topicConfig) APIErrorCode {
+	// Check queue arn is valid.
+	if s3Error := checkTopicARN(tConfig.TopicARN); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Unmarshals QueueARN into structured object.
+	topicARN := unmarshalTopicARN(tConfig.TopicARN)
+	// Validate if topicARN requested any of the known supported queues.
+	if !isValidTopic(topicARN) {
+		return ErrARNNotification
+	}
+
+	// Check if valid events are set in queue config.
+	if s3Error := checkEvents(tConfig.Events); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Check if valid filters are set in queue config.
+	if s3Error := checkFilterRules(tConfig.Filter.Key.FilterRules); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Success.
+	return ErrNone
+}
+
+// Validates all incoming queue configs, checkQueueConfig validates if the
+// input fields for each queues is not malformed and has valid configuration
+// information.  If validation fails bucket notifications are not enabled.
+func validateQueueConfigs(queueConfigs []queueConfig) APIErrorCode {
+	for _, qConfig := range queueConfigs {
+		if s3Error := checkQueueConfig(qConfig); s3Error != ErrNone {
+			return s3Error
+		}
+	}
+	// Success.
+	return ErrNone
+}
+
+// Validates all incoming topic configs, checkTopicConfig validates if the
+// input fields for each queues is not malformed and has valid configuration
+// information.  If validation fails bucket notifications are not enabled.
+func validateTopicConfigs(topicConfigs []topicConfig) APIErrorCode {
+	for _, tConfig := range topicConfigs {
+		if s3Error := checkTopicConfig(tConfig); s3Error != ErrNone {
+			return s3Error
+		}
+	}
+	// Success.
+	return ErrNone
+}
+
+// Validates all the bucket notification configuration for their validity,
+// if one of the config is malformed or has invalid data it is rejected.
+// Configuration is never applied partially.
+func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
+	if s3Error := validateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
+		return s3Error
+	}
+	if s3Error := validateTopicConfigs(nConfig.TopicConfigs); s3Error != ErrNone {
+		return s3Error
+	}
+
+	// Add validation for other configurations.
+	return ErrNone
+}
+
+// Unmarshals input value of AWS ARN format into minioTopic object.
+// Returned value represents minio topic type, currently supported are
+// - listen
+func unmarshalTopicARN(topicARN string) arnTopic {
+	topic := arnTopic{}
+	if !strings.HasPrefix(topicARN, minioTopic+serverConfig.GetRegion()+":") {
+		return topic
+	}
+	topicType := strings.TrimPrefix(topicARN, minioTopic+serverConfig.GetRegion()+":")
+	switch {
+	case strings.HasSuffix(topicType, snsTypeMinio):
+		topic.Type = snsTypeMinio
+	} // Add more topic here.
+	topic.AccountID = strings.TrimSuffix(topicType, ":"+topic.Type)
+	return topic
+}
+
+// Unmarshals input value of AWS ARN format into minioSqs object.
+// Returned value represents minio sqs types, currently supported are
+// - amqp
+// - elasticsearch
+// - redis
+func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
+	mSqs = arnSQS{}
+	if !strings.HasPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":") {
+		return mSqs
+	}
+	sqsType := strings.TrimPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":")
+	switch {
+	case strings.HasSuffix(sqsType, queueTypeAMQP):
+		mSqs.Type = queueTypeAMQP
+	case strings.HasSuffix(sqsType, queueTypeElastic):
+		mSqs.Type = queueTypeElastic
+	case strings.HasSuffix(sqsType, queueTypeRedis):
+		mSqs.Type = queueTypeRedis
+	} // Add more queues here.
+	mSqs.AccountID = strings.TrimSuffix(sqsType, ":"+mSqs.Type)
+	return mSqs
+}

cmd/bucket-notification-utils_test.go

@@ -0,0 +1,354 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "testing"
+
+// Tests filter name validation.
+func TestIsValidFilterName(t *testing.T) {
+	testCases := []struct {
+		filterName string
+		status     bool
+	}{
+		// Validate if 'prefix' is correct.
+		{
+			filterName: "prefix",
+			status:     true,
+		},
+		// Validate if 'suffix' is correct.
+		{
+			filterName: "suffix",
+			status:     true,
+		},
+		// Invalid filter name empty string should return false.
+		{
+			filterName: "",
+			status:     false,
+		},
+		// Invalid filter name random character should return false.
+		{
+			filterName: "unknown",
+			status:     false,
+		},
+	}
+
+	for i, testCase := range testCases {
+		status := isValidFilterName(testCase.filterName)
+		if testCase.status != status {
+			t.Errorf("Test %d: Expected \"%t\", got \"%t\"", i+1, testCase.status, status)
+		}
+	}
+}
+
+// Tests list of valid and invalid events.
+func TestValidEvents(t *testing.T) {
+	testCases := []struct {
+		events  []string
+		errCode APIErrorCode
+	}{
+		// Return error for unknown event element.
+		{
+			events: []string{
+				"s3:UnknownAPI",
+			},
+			errCode: ErrEventNotification,
+		},
+		// Return success for supported event.
+		{
+			events: []string{
+				"s3:ObjectCreated:Put",
+			},
+			errCode: ErrNone,
+		},
+		// Return success for supported events.
+		{
+			events: []string{
+				"s3:ObjectCreated:*",
+				"s3:ObjectRemoved:*",
+			},
+			errCode: ErrNone,
+		},
+		// Return error for empty event list.
+		{
+			events:  []string{""},
+			errCode: ErrEventNotification,
+		},
+	}
+
+	for i, testCase := range testCases {
+		errCode := checkEvents(testCase.events)
+		if testCase.errCode != errCode {
+			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
+		}
+	}
+}
+
+// Tests topic arn validation.
+func TestTopicARN(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases := []struct {
+		topicARN string
+		errCode  APIErrorCode
+	}{
+		// Valid minio topic with '1' account id.
+		{
+			topicARN: "arn:minio:sns:us-east-1:1:minio",
+			errCode:  ErrNone,
+		},
+		// Valid minio topic with '10' account id.
+		{
+			topicARN: "arn:minio:sns:us-east-1:10:minio",
+			errCode:  ErrNone,
+		},
+		// Invalid empty topic  arn.
+		{
+			topicARN: "",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid notification service type.
+		{
+			topicARN: "arn:minio:sqs:us-east-1:1:listen",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid region 'us-west-1' in queue arn.
+		{
+			topicARN: "arn:minio:sns:us-west-1:1:listen",
+			errCode:  ErrRegionNotification,
+		},
+		// Empty topic account id is invalid.
+		{
+			topicARN: "arn:minio:sns:us-east-1::listen",
+			errCode:  ErrARNNotification,
+		},
+		// Empty topic account name is invalid.
+		{
+			topicARN: "arn:minio:sns:us-east-1:10:",
+			errCode:  ErrARNNotification,
+		},
+		// Empty topic account id and account name is invalid.
+		{
+			topicARN: "arn:minio:sns:us-east-1::",
+			errCode:  ErrARNNotification,
+		},
+		// Missing topic id and separator missing at the end in topic arn.
+		{
+			topicARN: "arn:minio:sns:us-east-1:listen",
+			errCode:  ErrARNNotification,
+		},
+		// Missing topic id and empty string at the end in topic arn.
+		{
+			topicARN: "arn:minio:sns:us-east-1:",
+			errCode:  ErrARNNotification,
+		},
+	}
+
+	// Validate all topics.
+	for i, testCase := range testCases {
+		errCode := checkTopicARN(testCase.topicARN)
+		if testCase.errCode != errCode {
+			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
+		}
+	}
+}
+
+// Tests queue arn validation.
+func TestQueueARN(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases := []struct {
+		queueARN string
+		errCode  APIErrorCode
+	}{
+		// Valid redis queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:redis",
+			errCode:  ErrNone,
+		},
+		// Valid elasticsearch queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:elasticsearch",
+			errCode:  ErrNone,
+		},
+		// Valid amqp queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:amqp",
+			errCode:  ErrNone,
+		},
+		// Invalid empty queue arn.
+		{
+			queueARN: "",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid notification service type.
+		{
+			queueARN: "arn:minio:sns:us-east-1:1:listen",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid region 'us-west-1' in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:redis",
+			errCode:  ErrRegionNotification,
+		},
+		// Invalid queue name empty in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid queue id empty in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1::redis",
+			errCode:  ErrARNNotification,
+		},
+		// Invalid queue id and queue name empty in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1::",
+			errCode:  ErrARNNotification,
+		},
+		// Missing queue id and separator missing at the end in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:amqp",
+			errCode:  ErrARNNotification,
+		},
+		// Missing queue id and empty string at the end in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:",
+			errCode:  ErrARNNotification,
+		},
+	}
+
+	// Validate all tests for queue arn.
+	for i, testCase := range testCases {
+		errCode := checkQueueARN(testCase.queueARN)
+		if testCase.errCode != errCode {
+			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
+		}
+	}
+}
+
+// Test unmarshal topic arn.
+func TestUnmarshalTopicARN(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases := []struct {
+		topicARN string
+		Type     string
+	}{
+		// Valid minio topic arn.
+		{
+			topicARN: "arn:minio:sns:us-east-1:1:listen",
+			Type:     "listen",
+		},
+		// Invalid empty topic arn.
+		{
+			topicARN: "",
+			Type:     "",
+		},
+		// Invalid region 'us-west-1' in topic arn.
+		{
+			topicARN: "arn:minio:sns:us-west-1:1:listen",
+			Type:     "",
+		},
+		// Partial topic arn.
+		{
+			topicARN: "arn:minio:sns:",
+			Type:     "",
+		},
+		// Invalid topic service value.
+		{
+			topicARN: "arn:minio:sns:us-east-1:1:*",
+			Type:     "",
+		},
+	}
+
+	for i, testCase := range testCases {
+		topic := unmarshalTopicARN(testCase.topicARN)
+		if testCase.Type != topic.Type {
+			t.Errorf("Test %d: Expected \"%s\", got \"%s\"", i+1, testCase.Type, topic.Type)
+		}
+	}
+}
+
+// Test unmarshal queue arn.
+func TestUnmarshalSqsARN(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(rootPath)
+
+	testCases := []struct {
+		queueARN string
+		Type     string
+	}{
+		// Valid redis queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:redis",
+			Type:     "redis",
+		},
+		// Valid elasticsearch queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:elasticsearch",
+			Type:     "elasticsearch",
+		},
+		// Valid amqp queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:amqp",
+			Type:     "amqp",
+		},
+		// Invalid empty queue arn.
+		{
+			queueARN: "",
+			Type:     "",
+		},
+		// Invalid region 'us-west-1' in queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:redis",
+			Type:     "",
+		},
+		// Partial queue arn.
+		{
+			queueARN: "arn:minio:sqs:",
+			Type:     "",
+		},
+		// Invalid queue service value.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:*",
+			Type:     "",
+		},
+	}
+
+	for i, testCase := range testCases {
+		mSqs := unmarshalSqsARN(testCase.queueARN)
+		if testCase.Type != mSqs.Type {
+			t.Errorf("Test %d: Expected \"%s\", got \"%s\"", i+1, testCase.Type, mSqs.Type)
+		}
+	}
+
+}

cmd/bucket-policy-handlers.go

@@ -0,0 +1,288 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	mux "github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/wildcard"
+)
+
+// maximum supported access policy size.
+const maxAccessPolicySize = 20 * 1024 // 20KiB.
+
+// Verify if a given action is valid for the url path based on the
+// existing bucket access policy.
+func bucketPolicyEvalStatements(action string, resource string, conditions map[string]string, statements []policyStatement) bool {
+	for _, statement := range statements {
+		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
+			if statement.Effect == "Allow" {
+				return true
+			}
+			// Do not uncomment kept here for readability.
+			// else statement.Effect == "Deny"
+			return false
+		}
+	}
+	// None match so deny.
+	return false
+}
+
+// Verify if action, resource and conditions match input policy statement.
+func bucketPolicyMatchStatement(action string, resource string, conditions map[string]string, statement policyStatement) bool {
+	// Verify if action matches.
+	if bucketPolicyActionMatch(action, statement) {
+		// Verify if resource matches.
+		if bucketPolicyResourceMatch(resource, statement) {
+			// Verify if condition matches.
+			if bucketPolicyConditionMatch(conditions, statement) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Verify if given action matches with policy statement.
+func bucketPolicyActionMatch(action string, statement policyStatement) bool {
+	for _, policyAction := range statement.Actions {
+		if matched := actionMatch(policyAction, action); matched {
+			return true
+		}
+	}
+	return false
+}
+
+// Match function matches wild cards in 'pattern' for resource.
+func resourceMatch(pattern, resource string) bool {
+	return wildcard.MatchExtended(pattern, resource)
+}
+
+// Match function matches wild cards in 'pattern' for action.
+func actionMatch(pattern, action string) bool {
+	return wildcard.Match(pattern, action)
+}
+
+// Verify if given resource matches with policy statement.
+func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
+	for _, resourcep := range statement.Resources {
+		// the resource rule for object could contain "*" wild card.
+		// the requested object can be given access based on the already set bucket policy if
+		// the match is successful.
+		// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html .
+		if matched := resourceMatch(resourcep, resource); matched {
+			return true
+		}
+	}
+	return false
+}
+
+// Verify if given condition matches with policy statement.
+func bucketPolicyConditionMatch(conditions map[string]string, statement policyStatement) bool {
+	// Supports following conditions.
+	// - StringEquals
+	// - StringNotEquals
+	//
+	// Supported applicable condition keys for each conditions.
+	// - s3:prefix
+	// - s3:max-keys
+	var conditionMatches = true
+	for condition, conditionKeys := range statement.Conditions {
+		if condition == "StringEquals" {
+			if conditionKeys["s3:prefix"] != conditions["prefix"] {
+				conditionMatches = false
+				break
+			}
+			if conditionKeys["s3:max-keys"] != conditions["max-keys"] {
+				conditionMatches = false
+				break
+			}
+		} else if condition == "StringNotEquals" {
+			if conditionKeys["s3:prefix"] == conditions["prefix"] {
+				conditionMatches = false
+				break
+			}
+			if conditionKeys["s3:max-keys"] == conditions["max-keys"] {
+				conditionMatches = false
+				break
+			}
+		}
+	}
+	return conditionMatches
+}
+
+// PutBucketPolicyHandler - PUT Bucket policy
+// -----------------
+// This implementation of the PUT operation uses the policy
+// subresource to add to or replace a policy on a bucket
+func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// If Content-Length is unknown or zero, deny the
+	// request. PutBucketPolicy always needs a Content-Length if
+	// incoming request is not chunked.
+	if !contains(r.TransferEncoding, "chunked") {
+		if r.ContentLength == -1 || r.ContentLength == 0 {
+			writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+			return
+		}
+		// If Content-Length is greater than maximum allowed policy size.
+		if r.ContentLength > maxAccessPolicySize {
+			writeErrorResponse(w, r, ErrEntityTooLarge, r.URL.Path)
+			return
+		}
+	}
+
+	// Read access policy up to maxAccessPolicySize.
+	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+	// bucket policies are limited to 20KB in size, using a limit reader.
+	policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	if err != nil {
+		errorIf(err, "Unable to read from client.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	// Parse bucket policy.
+	var policy = &bucketPolicy{}
+	err = parseBucketPolicy(bytes.NewReader(policyBytes), policy)
+	if err != nil {
+		errorIf(err, "Unable to parse bucket policy.")
+		writeErrorResponse(w, r, ErrInvalidPolicyDocument, r.URL.Path)
+		return
+	}
+
+	// Parse check bucket policy.
+	if s3Error := checkBucketPolicyResources(bucket, policy); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Save bucket policy.
+	if err = writeBucketPolicy(bucket, api.ObjectAPI, bytes.NewReader(policyBytes), int64(len(policyBytes))); err != nil {
+		errorIf(err, "Unable to write bucket policy.")
+		switch err.(type) {
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+
+	// Set the bucket policy in memory.
+	globalBucketPolicies.SetBucketPolicy(bucket, policy)
+
+	// Success.
+	writeSuccessNoContent(w)
+}
+
+// DeleteBucketPolicyHandler - DELETE Bucket policy
+// -----------------
+// This implementation of the DELETE operation uses the policy
+// subresource to add to remove a policy on a bucket.
+func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// Delete bucket access policy.
+	if err := removeBucketPolicy(bucket, api.ObjectAPI); err != nil {
+		errorIf(err, "Unable to remove bucket policy.")
+		switch err.(type) {
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		case BucketPolicyNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+
+	// Remove bucket policy.
+	globalBucketPolicies.RemoveBucketPolicy(bucket)
+
+	// Success.
+	writeSuccessNoContent(w)
+}
+
+// GetBucketPolicyHandler - GET Bucket policy
+// -----------------
+// This operation uses the policy
+// subresource to return the policy of a specified bucket.
+func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// Read bucket access policy.
+	policy, err := readBucketPolicy(bucket, api.ObjectAPI)
+	if err != nil {
+		errorIf(err, "Unable to read bucket policy.")
+		switch err.(type) {
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		case BucketPolicyNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+
+	// Write to client.
+	fmt.Fprint(w, policy)
+}

cmd/bucket-policy-handlers_test.go

@@ -0,0 +1,547 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+// Tests validate Bucket policy resource matcher.
+func TestBucketPolicyResourceMatch(t *testing.T) {
+
+	// generates statement with given resource..
+	generateStatement := func(resource string) policyStatement {
+		statement := policyStatement{}
+		statement.Resources = []string{resource}
+		return statement
+	}
+
+	// generates resource prefix.
+	generateResource := func(bucketName, objectName string) string {
+		return AWSResourcePrefix + bucketName + "/" + objectName
+	}
+
+	testCases := []struct {
+		resourceToMatch       string
+		statement             policyStatement
+		expectedResourceMatch bool
+	}{
+		// Test case 1-4.
+		// Policy with resource ending with bucket/* allows access to all objects inside the given bucket.
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/*")), true},
+		// Test case - 5.
+		// Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt.
+		{generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), false},
+		// Test case - 6.
+		// Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt.
+		{generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true},
+		// Test case - 7.
+		// Policy with resource ending with bucket/oo* allows access to all subfolders starting with "oo" inside given bucket.
+		{generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/oo*")), true},
+		// Test case - 8.
+		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false},
+		// Test case - 9.
+		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false},
+		// Test case - 10.
+		// Proves that the name space is flat.
+		{generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix,
+			"minio-bucket"+"/*/India/*/Bihar")), true},
+		// Test case - 11.
+		// Proves that the name space is flat.
+		{generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", AWSResourcePrefix,
+			"minio-bucket"+"/*/India/*/Bihar/*")), true},
+	}
+	for i, testCase := range testCases {
+		actualResourceMatch := bucketPolicyResourceMatch(testCase.resourceToMatch, testCase.statement)
+		if testCase.expectedResourceMatch != actualResourceMatch {
+			t.Errorf("Test %d: Expected Resource match to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResourceMatch, actualResourceMatch)
+		}
+	}
+}
+
+// TestBucketPolicyActionMatch - Test validates whether given action on the
+// bucket/object matches the allowed actions in policyStatement.
+// This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
+// The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
+func TestBucketPolicyActionMatch(t *testing.T) {
+	bucketName := getRandomBucketName()
+	objectPrefix := "test-object"
+
+	testCases := []struct {
+		action         string
+		statement      policyStatement
+		expectedResult bool
+	}{
+		// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
+		// fetching of bucket location on an Anonymous/unsigned request.
+
+		//r ead-write bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 1).
+		{"s3:GetBucketLocation", getReadWriteBucketStatement(bucketName, objectPrefix), true},
+		//	write-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 2).
+		{"s3:GetBucketLocation", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
+		//	read-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 3).
+		{"s3:GetBucketLocation", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
+
+		// Any of the Object level access permissions shouldn't allow for GetBucketLocation operation  on an Anonymous/unsigned request (Test cases 4-6).
+		{"s3:GetBucketLocation", getReadWriteObjectStatement(bucketName, objectPrefix), false},
+		{"s3:GetBucketLocation", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+		{"s3:GetBucketLocation", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:ListBucketMultipartUploads is the action necessary to be present in the bucket policy to allow
+		// Listing of multipart uploads in a given bucket for an Anonymous/unsigned request.
+
+		//read-write bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 7).
+		{"s3:ListBucketMultipartUploads", getReadWriteBucketStatement(bucketName, objectPrefix), true},
+		//	write-only bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 8).
+		{"s3:ListBucketMultipartUploads", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
+		// read-only bucket policy is expected to not allow ListBucketMultipartUploads operation on an anonymous request (Test case 9).
+		// the allowed actions in read-only bucket statement are  "s3:GetBucketLocation","s3:ListBucket",
+		// this should not allow for ListBucketMultipartUploads operations.
+		{"s3:ListBucketMultipartUploads", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Any of the object level policy will not allow for s3:ListBucketMultipartUploads (Test cases 10-12).
+		{"s3:ListBucketMultipartUploads", getReadWriteObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucketMultipartUploads", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucketMultipartUploads", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:ListBucket is the action necessary to be present in the bucket policy to allow
+		// listing of all objects inside a given bucket on an Anonymous/unsigned request.
+
+		// Cases for testing ListBucket access for different Bucket level access permissions.
+		// read-only bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 13).
+		{"s3:ListBucket", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
+		// read-write bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 14).
+		{"s3:ListBucket", getReadWriteBucketStatement(bucketName, objectPrefix), true},
+		// write-only  bucket policy is expected to not allow ListBucket operation on an anonymous request (Test case 15).
+		// the allowed actions in write-only  bucket statement are "s3:GetBucketLocation",	"s3:ListBucketMultipartUploads",
+		// this should not allow for ListBucket operations.
+		{"s3:ListBucket", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing ListBucket access for different Object level access permissions (Test cases 16-18).
+		// Any of the Object level access permissions shouldn't allow for ListBucket operation  on an Anonymous/unsigned request.
+		{"s3:ListBucket", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucket", getReadWriteObjectStatement(bucketName, objectPrefix), false},
+		{"s3:ListBucket", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:DeleteObject is the action necessary to be present in the bucket policy to allow
+		// deleting/removal of  objects inside a given bucket for an Anonymous/unsigned request.
+
+		// Cases for testing DeleteObject access for different Bucket level access permissions (Test cases 19-21).
+		// Any of the Bucket level access permissions shouldn't allow for DeleteObject operation  on an Anonymous/unsigned request.
+		{"s3:DeleteObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:DeleteObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:DeleteObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing DeleteObject access for different Object level access permissions (Test cases 22).
+		// read-only bucket policy is expected to not allow Delete Object operation on an anonymous request.
+		{"s3:DeleteObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write bucket policy is expected to allow Delete Bucket operation on an anonymous request (Test cases 23).
+		{"s3:DeleteObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  bucket policy is expected to allow Delete Object operation on an anonymous request (Test cases 24).
+		{"s3:DeleteObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+
+		// s3:AbortMultipartUpload is the action necessary to be present in the bucket policy to allow
+		// cancelling or abortion of an already initiated multipart upload operation for an Anonymous/unsigned request.
+
+		// Cases for testing AbortMultipartUpload access for different Bucket level access permissions (Test cases 25-27).
+		// Any of the Bucket level access permissions shouldn't allow for AbortMultipartUpload operation on an Anonymous/unsigned request.
+		{"s3:AbortMultipartUpload", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:AbortMultipartUpload", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:AbortMultipartUpload", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing AbortMultipartUpload access for different Object level access permissions.
+		// read-only object policy is expected to not allow AbortMultipartUpload operation on an anonymous request (Test case 28).
+		{"s3:AbortMultipartUpload", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 29).
+		{"s3:AbortMultipartUpload", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 30).
+		{"s3:AbortMultipartUpload", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+
+		// s3:PutObject is the action necessary to be present in the bucket policy to allow
+		// uploading of an object for an Anonymous/unsigned request.
+
+		// Cases for testing PutObject access for different Bucket level access permissions (Test cases 31-33).
+		// Any of the Bucket level access permissions shouldn't allow for PutObject operation on an Anonymous/unsigned request.
+		{"s3:PutObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:PutObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:PutObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing PutObject access for different Object level access permissions.
+		// read-only object policy is expected to not allow PutObject operation on an anonymous request (Test case 34).
+		{"s3:PutObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write object policy is expected to allow PutObject operation on an anonymous request (Test case 35).
+		{"s3:PutObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  object policy is expected to allow PutObject operation on an anonymous request (Test case 36).
+		{"s3:PutObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+
+		// s3:GetObject is the action necessary to be present in the bucket policy to allow
+		// downloading of an object for an Anonymous/unsigned request.
+
+		// Cases for testing GetObject access for different Bucket level access permissions (Test cases 37-39).
+		// Any of the Bucket level access permissions shouldn't allow for GetObject operation on an Anonymous/unsigned request.
+		{"s3:GetObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:GetObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:GetObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// Cases for testing GetObject access for different Object level access permissions.
+		// read-only bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 40).
+		{"s3:GetObject", getReadOnlyObjectStatement(bucketName, objectPrefix), true},
+		// read-write bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 41).
+		{"s3:GetObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  bucket policy is expected to not allow  downloading of an Object on an anonymous request (Test case 42).
+		{"s3:GetObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
+
+		// s3:ListMultipartUploadParts is the action necessary to be present in the bucket policy to allow
+		// Listing of uploaded parts for an Anonymous/unsigned request.
+
+		// Any of the Bucket level access permissions shouldn't allow for ListMultipartUploadParts operation on an Anonymous/unsigned request.
+		// read-only bucket policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test cases 43-45).
+		{"s3:ListMultipartUploadParts", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
+		{"s3:ListMultipartUploadParts", getReadWriteBucketStatement(bucketName, objectPrefix), false},
+		{"s3:ListMultipartUploadParts", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
+
+		// read-only object policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test case 46).
+		{"s3:ListMultipartUploadParts", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
+		// read-write object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 47).
+		{"s3:ListMultipartUploadParts", getReadWriteObjectStatement(bucketName, objectPrefix), true},
+		// write-only  object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 48).
+		{"s3:ListMultipartUploadParts", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+	}
+	for i, testCase := range testCases {
+		actualResult := bucketPolicyActionMatch(testCase.action, testCase.statement)
+		if testCase.expectedResult != actualResult {
+			t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResult, actualResult)
+		}
+	}
+}
+
+// Wrapper for calling Put Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+func TestPutBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testPutBucketPolicyHandler)
+}
+
+// testPutBucketPolicyHandler - Test for Bucket policy end point.
+// TODO: Add exhaustive cases with various combination of statement fields.
+func testPutBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	initBucketPolicies(obj)
+
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+	// Create bucket.
+	err := obj.MakeBucket(bucketName)
+	if err != nil {
+		// failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+	// Register the API end points with XL/FS object layer.
+	apiRouter := initTestAPIEndPoints(obj, []string{"PutBucketPolicy"})
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	credentials := serverConfig.GetCredential()
+
+	// template for constructing HTTP request body for PUT bucket policy.
+	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
+
+	// test cases with sample input and expected output.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, http.StatusNoContent},
+	}
+
+	// Iterating over the test cases, calling the function under test and asserting the response.
+	for i, testCase := range testCases {
+		// obtain the put bucket policy request body.
+		bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testCase.bucketName, testCase.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		req, err := newTestSignedRequest("PUT", getPutPolicyURL("", testCase.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testCase.accessKey, testCase.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testCase.expectedRespStatus {
+			t.Errorf("Test %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code)
+		}
+	}
+}
+
+// Wrapper for calling Get Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+func TestGetBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testGetBucketPolicyHandler)
+}
+
+// testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket.
+// TODO: Add exhaustive cases with various combination of statement fields.
+func testGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	initBucketPolicies(obj)
+
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+	// Create bucket.
+	err := obj.MakeBucket(bucketName)
+	if err != nil {
+		// failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+	// Register the API end points with XL/FS object layer.
+	// Registering only the PutBucketPolicy and GetBucketPolicy handlers.
+	apiRouter := initTestAPIEndPoints(obj, []string{"PutBucketPolicy", "GetBucketPolicy"})
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	credentials := serverConfig.GetCredential()
+
+	// template for constructing HTTP request body for PUT bucket policy.
+	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
+
+	// Writing bucket policy before running test on GetBucketPolicy.
+	putTestPolicies := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, http.StatusNoContent},
+	}
+
+	// Iterating over the cases and writing the bucket policy.
+	// its required to write the policies first before running tests on GetBucketPolicy.
+	for i, testPolicy := range putTestPolicies {
+		// obtain the put bucket policy request body.
+		bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		req, err := newTestSignedRequest("PUT", getPutPolicyURL("", testPolicy.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testPolicy.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, rec.Code)
+		}
+	}
+
+	// test cases with inputs and expected result for GetBucketPolicyHandler.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected output.
+		expectedBucketPolicy string
+		expectedRespStatus   int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, bucketPolicyTemplate, http.StatusOK},
+	}
+	// Iterating over the cases, fetching the policy and validating the response.
+	for i, testCase := range testCases {
+		// expected bucket policy json string.
+		expectedBucketPolicyStr := fmt.Sprintf(testCase.expectedBucketPolicy, testCase.bucketName, testCase.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		req, err := newTestSignedRequest("GET", getGetPolicyURL("", testCase.bucketName),
+			0, nil, testCase.accessKey, testCase.secretKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler, GetBucketPolicyHandler handles the request.
+		apiRouter.ServeHTTP(rec, req)
+		// Assert the response code with the expected status.
+		if rec.Code != testCase.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code)
+		}
+		// read the response body.
+		bucketPolicyReadBuf, err := ioutil.ReadAll(rec.Body)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Verify whether the bucket policy fetched is same as the one inserted.
+		if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+			t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
+		}
+	}
+}
+
+// Wrapper for calling Delete Bucket Policy HTTP handler tests for both XL multiple disks and single node setup.
+func TestDeleteBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testDeleteBucketPolicyHandler)
+}
+
+// testDeleteBucketPolicyHandler - Test for Delete bucket policy end point.
+// TODO: Add exhaustive cases with various combination of statement fields.
+func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	initBucketPolicies(obj)
+
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+	// Create bucket.
+	err := obj.MakeBucket(bucketName)
+	if err != nil {
+		// failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+
+	// Register the API end points with XL/FS object layer.
+	// Registering PutBucketPolicy and DeleteBucketPolicy handlers.
+	apiRouter := initTestAPIEndPoints(obj, []string{"PutBucketPolicy", "DeleteBucketPolicy"})
+
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	credentials := serverConfig.GetCredential()
+
+	// template for constructing HTTP request body for PUT bucket policy.
+	bucketPolicyTemplate := `{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "s3:GetBucketLocation",
+                "s3:ListBucket"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s"
+            ]
+        },
+        {
+            "Action": [
+                "s3:GetObject"
+            ],
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "*"
+                ]
+            },
+            "Resource": [
+                "arn:aws:s3:::%s/this*"
+            ]
+        }
+    ]
+}`
+
+	// Writing bucket policy before running test on DeleteBucketPolicy.
+	putTestPolicies := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, http.StatusNoContent},
+	}
+
+	// Iterating over the cases and writing the bucket policy.
+	// its required to write the policies first before running tests on GetBucketPolicy.
+	for i, testPolicy := range putTestPolicies {
+		// obtain the put bucket policy request body.
+		bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName)
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for PUT bucket policy endpoint.
+		req, err := newTestSignedRequest("PUT", getPutPolicyURL("", testPolicy.bucketName),
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler.
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != testPolicy.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, rec.Code)
+		}
+	}
+	// testcases with input and expected output for DeleteBucketPolicyHandler.
+	testCases := []struct {
+		bucketName string
+		accessKey  string
+		secretKey  string
+		// expected Response.
+		expectedRespStatus int
+	}{
+		{bucketName, credentials.AccessKeyID, credentials.SecretAccessKey, http.StatusNoContent},
+	}
+	// Iterating over the cases and deleting the bucket policy and then asserting response.
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for Delete bucket policy endpoint.
+		req, err := newTestSignedRequest("DELETE", getDeletePolicyURL("", testCase.bucketName),
+			0, nil, testCase.accessKey, testCase.secretKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler, DeleteBucketPolicyHandler  handles the request.
+		apiRouter.ServeHTTP(rec, req)
+		// Assert the response code with the expected status.
+		if rec.Code != testCase.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code)
+		}
+
+	}
+}

cmd/bucket-policy-migrate.go

@@ -0,0 +1,83 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const policyJSON = "policy.json"
+
+func getBucketFromPolicyPath(oldPolicyPath string) string {
+	bucketPrefix, _ := filepath.Split(oldPolicyPath)
+	_, bucketName := filepath.Split(strings.TrimSuffix(bucketPrefix, slashSeparator))
+	return bucketName
+}
+
+func cleanupOldBucketPolicyConfigs() error {
+	// Get old bucket policy config directory.
+	oldBucketsConfigDir, err := getOldBucketsConfigPath()
+	fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy")
+
+	// Recursively remove configDir/buckets/ - old bucket policy config location.
+	// N B This is called only if all bucket policies were successfully migrated.
+	return os.RemoveAll(oldBucketsConfigDir)
+}
+
+func migrateBucketPolicyConfig(objAPI ObjectLayer) error {
+	// Get old bucket policy config directory.
+	oldBucketsConfigDir, err := getOldBucketsConfigPath()
+	fatalIf(err, "Unable to fetch buckets config path to migrate bucket policy")
+
+	// Check if config directory holding bucket policy exists before
+	// migration.
+	_, err = os.Stat(oldBucketsConfigDir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	// WalkFunc that migrates access-policy.json to
+	// .minio.sys/buckets/bucketName/policy.json on all disks.
+	migrateBucketPolicy := func(policyPath string, fileInfo os.FileInfo, err error) error {
+		// policyFile - e.g /configDir/sample-bucket/access-policy.json
+		if err != nil {
+			return err
+		}
+		// Skip entries that aren't bucket policy files.
+		if fileInfo.Name() != "access-policy.json" {
+			return nil
+		}
+		// Get bucketName from old policy file path.
+		bucketName := getBucketFromPolicyPath(policyPath)
+		// Read bucket policy config from old location.
+		policyBytes, err := ioutil.ReadFile(policyPath)
+		fatalIf(err, "Unable to read bucket policy to migrate bucket policy", policyPath)
+		newPolicyPath := retainSlash(bucketConfigPrefix) + retainSlash(bucketName) + policyJSON
+		// Erasure code the policy config to all the disks.
+		_, err = objAPI.PutObject(minioMetaBucket, newPolicyPath, int64(len(policyBytes)), bytes.NewReader(policyBytes), nil)
+		fatalIf(err, "Unable to write bucket policy during migration.", newPolicyPath)
+		return nil
+	}
+	return filepath.Walk(oldBucketsConfigDir, migrateBucketPolicy)
+}

cmd/bucket-policy-parser.go

@@ -0,0 +1,343 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package cmd This file implements AWS Access Policy Language parser in
+// accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"sort"
+	"strings"
+)
+
+const (
+	// AWSResourcePrefix - bucket policy resource prefix.
+	AWSResourcePrefix = "arn:aws:s3:::"
+)
+
+// supportedActionMap - lists all the actions supported by minio.
+var supportedActionMap = map[string]struct{}{
+	"*":                             {},
+	"s3:*":                          {},
+	"s3:GetObject":                  {},
+	"s3:ListBucket":                 {},
+	"s3:PutObject":                  {},
+	"s3:GetBucketLocation":          {},
+	"s3:DeleteObject":               {},
+	"s3:AbortMultipartUpload":       {},
+	"s3:ListBucketMultipartUploads": {},
+	"s3:ListMultipartUploadParts":   {},
+}
+
+// supported Conditions type.
+var supportedConditionsType = map[string]struct{}{
+	"StringEquals":    {},
+	"StringNotEquals": {},
+}
+
+// Validate s3:prefix, s3:max-keys are present if not
+// supported keys for the conditions.
+var supportedConditionsKey = map[string]struct{}{
+	"s3:prefix":   {},
+	"s3:max-keys": {},
+}
+
+// User - canonical users list.
+type policyUser struct {
+	AWS []string
+}
+
+// Statement - minio policy statement
+type policyStatement struct {
+	Sid        string
+	Effect     string
+	Principal  policyUser                   `json:"Principal"`
+	Actions    []string                     `json:"Action"`
+	Resources  []string                     `json:"Resource"`
+	Conditions map[string]map[string]string `json:"Condition,omitempty"`
+}
+
+// bucketPolicy - collection of various bucket policy statements.
+type bucketPolicy struct {
+	Version    string            // date in 0000-00-00 format
+	Statements []policyStatement `json:"Statement"`
+}
+
+// Stringer implementation for the bucket policies.
+func (b bucketPolicy) String() string {
+	bbytes, err := json.Marshal(&b)
+	if err != nil {
+		errorIf(err, "Unable to unmarshal bucket policy into JSON %#v", b)
+		return ""
+	}
+	return string(bbytes)
+}
+
+// supportedEffectMap - supported effects.
+var supportedEffectMap = map[string]struct{}{
+	"Allow": {},
+	"Deny":  {},
+}
+
+// isValidActions - are actions valid.
+func isValidActions(actions []string) (err error) {
+	// Statement actions cannot be empty.
+	if len(actions) == 0 {
+		err = errors.New("Action list cannot be empty.")
+		return err
+	}
+	for _, action := range actions {
+		if _, ok := supportedActionMap[action]; !ok {
+			err = errors.New("Unsupported action found: ‘" + action + "’, please validate your policy document.")
+			return err
+		}
+	}
+	return nil
+}
+
+// isValidEffect - is effect valid.
+func isValidEffect(effect string) error {
+	// Statement effect cannot be empty.
+	if len(effect) == 0 {
+		err := errors.New("Policy effect cannot be empty.")
+		return err
+	}
+	_, ok := supportedEffectMap[effect]
+	if !ok {
+		err := errors.New("Unsupported Effect found: ‘" + effect + "’, please validate your policy document.")
+		return err
+	}
+	return nil
+}
+
+// isValidResources - are valid resources.
+func isValidResources(resources []string) (err error) {
+	// Statement resources cannot be empty.
+	if len(resources) == 0 {
+		err = errors.New("Resource list cannot be empty.")
+		return err
+	}
+	for _, resource := range resources {
+		if !strings.HasPrefix(resource, AWSResourcePrefix) {
+			err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document.")
+			return err
+		}
+		resourceSuffix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+		if len(resourceSuffix) == 0 || strings.HasPrefix(resourceSuffix, "/") {
+			err = errors.New("Invalid resource style found: ‘" + resource + "’, please validate your policy document.")
+			return err
+		}
+	}
+	return nil
+}
+
+// isValidPrincipals - are valid principals.
+func isValidPrincipals(principals []string) (err error) {
+	// Statement principal should have a value.
+	if len(principals) == 0 {
+		err = errors.New("Principal cannot be empty.")
+		return err
+	}
+	for _, principal := range principals {
+		// Minio does not support or implement IAM, "*" is the only valid value.
+		// Amazon s3 doc on principals: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
+		if principal != "*" {
+			err = fmt.Errorf("Unsupported principal style found: ‘%s’, please validate your policy document.", principal)
+			return err
+		}
+	}
+	return nil
+}
+
+// isValidConditions - are valid conditions.
+func isValidConditions(conditions map[string]map[string]string) (err error) {
+	// Returns true if string 'a' is found in the list.
+	findString := func(a string, list []string) bool {
+		for _, b := range list {
+			if b == a {
+				return true
+			}
+		}
+		return false
+	}
+	conditionKeyVal := make(map[string][]string)
+	// Verify conditions should be valid.
+	// Validate if stringEquals, stringNotEquals are present
+	// if not throw an error.
+	for conditionType := range conditions {
+		_, validType := supportedConditionsType[conditionType]
+		if !validType {
+			err = fmt.Errorf("Unsupported condition type '%s', please validate your policy document.", conditionType)
+			return err
+		}
+		for key := range conditions[conditionType] {
+			_, validKey := supportedConditionsKey[key]
+			if !validKey {
+				err = fmt.Errorf("Unsupported condition key '%s', please validate your policy document.", conditionType)
+				return err
+			}
+			conditionArray, ok := conditionKeyVal[key]
+			if ok && findString(conditions[conditionType][key], conditionArray) {
+				err = fmt.Errorf("Ambigious condition values for key '%s', please validate your policy document.", key)
+				return err
+			}
+			conditionKeyVal[key] = append(conditionKeyVal[key], conditions[conditionType][key])
+		}
+	}
+	return nil
+}
+
+// List of actions for which prefixes are not allowed.
+var invalidPrefixActions = map[string]struct{}{
+	"s3:GetBucketLocation":          {},
+	"s3:ListBucket":                 {},
+	"s3:ListBucketMultipartUploads": {},
+	// Add actions which do not honor prefixes.
+}
+
+// resourcePrefix - provides the prefix removing any wildcards.
+func resourcePrefix(resource string) string {
+	if strings.HasSuffix(resource, "*") {
+		resource = strings.TrimSuffix(resource, "*")
+	}
+	return path.Clean(resource)
+}
+
+// checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
+// First valation of Resources done for given set of Actions.
+// Later its validated for recursive Resources.
+func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIErrorCode {
+	// Validate statements for special actions and collect resources
+	// for others to validate nesting.
+	var resourceMap = make(map[string]struct{})
+	for _, statement := range bucketPolicy.Statements {
+		for _, action := range statement.Actions {
+			for _, resource := range statement.Resources {
+				resourcePrefix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+				if _, ok := invalidPrefixActions[action]; ok {
+					// Resource prefix is not equal to bucket for
+					// prefix invalid actions, reject them.
+					if resourcePrefix != bucket {
+						return ErrMalformedPolicy
+					}
+				} else {
+					// For all other actions validate if resourcePrefix begins
+					// with bucket name, if not reject them.
+					if strings.Split(resourcePrefix, "/")[0] != bucket {
+						return ErrMalformedPolicy
+					}
+					// All valid resources collect them separately to verify nesting.
+					resourceMap[resourcePrefix] = struct{}{}
+				}
+			}
+		}
+	}
+
+	var resources []string
+	for resource := range resourceMap {
+		resources = append(resources, resourcePrefix(resource))
+	}
+
+	// Sort strings as shorter first.
+	sort.Strings(resources)
+
+	for len(resources) > 1 {
+		var resource string
+		resource, resources = resources[0], resources[1:]
+		// Loop through all resources, if one of them matches with
+		// previous shorter one, it means we have detected
+		// nesting. Reject such rules.
+		for _, otherResource := range resources {
+			// Common prefix reject such rules.
+			if strings.HasPrefix(otherResource, resource) {
+				return ErrPolicyNesting
+			}
+		}
+	}
+
+	// No errors found.
+	return ErrNone
+}
+
+// parseBucketPolicy - parses and validates if bucket policy is of
+// proper JSON and follows allowed restrictions with policy standards.
+func parseBucketPolicy(bucketPolicyReader io.Reader, policy *bucketPolicy) (err error) {
+	// Parse bucket policy reader.
+	decoder := json.NewDecoder(bucketPolicyReader)
+	if err = decoder.Decode(&policy); err != nil {
+		return err
+	}
+
+	// Policy version cannot be empty.
+	if len(policy.Version) == 0 {
+		err = errors.New("Policy version cannot be empty.")
+		return err
+	}
+
+	// Policy statements cannot be empty.
+	if len(policy.Statements) == 0 {
+		err = errors.New("Policy statement cannot be empty.")
+		return err
+	}
+
+	// Loop through all policy statements and validate entries.
+	for _, statement := range policy.Statements {
+		// Statement effect should be valid.
+		if err := isValidEffect(statement.Effect); err != nil {
+			return err
+		}
+		// Statement principal should be supported format.
+		if err := isValidPrincipals(statement.Principal.AWS); err != nil {
+			return err
+		}
+		// Statement actions should be valid.
+		if err := isValidActions(statement.Actions); err != nil {
+			return err
+		}
+		// Statement resources should be valid.
+		if err := isValidResources(statement.Resources); err != nil {
+			return err
+		}
+		// Statement conditions should be valid.
+		if err := isValidConditions(statement.Conditions); err != nil {
+			return err
+		}
+	}
+
+	// Separate deny and allow statements, so that we can apply deny
+	// statements in the beginning followed by Allow statements.
+	var denyStatements []policyStatement
+	var allowStatements []policyStatement
+	for _, statement := range policy.Statements {
+		if statement.Effect == "Deny" {
+			denyStatements = append(denyStatements, statement)
+			continue
+		}
+		// else if statement.Effect == "Allow"
+		allowStatements = append(allowStatements, statement)
+	}
+
+	// Deny statements are enforced first once matched.
+	policy.Statements = append(denyStatements, allowStatements...)
+
+	// Return successfully parsed policy structure.
+	return nil
+}

cmd/bucket-policy-parser_test.go

@@ -0,0 +1,695 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"testing"
+)
+
+var (
+	readWriteBucketActions = []string{
+		"s3:GetBucketLocation",
+		"s3:ListBucket",
+		"s3:ListBucketMultipartUploads",
+		// Add more bucket level read-write actions here.
+	}
+	readWriteObjectActions = []string{
+		"s3:AbortMultipartUpload",
+		"s3:DeleteObject",
+		"s3:GetObject",
+		"s3:ListMultipartUploadParts",
+		"s3:PutObject",
+		// Add more object level read-write actions here.
+	}
+)
+
+// Write only actions.
+var (
+	writeOnlyBucketActions = []string{
+		"s3:GetBucketLocation",
+		"s3:ListBucketMultipartUploads",
+		// Add more bucket level write actions here.
+	}
+	writeOnlyObjectActions = []string{
+		"s3:AbortMultipartUpload",
+		"s3:DeleteObject",
+		"s3:ListMultipartUploadParts",
+		"s3:PutObject",
+		// Add more object level write actions here.
+	}
+)
+
+// Read only actions.
+var (
+	readOnlyBucketActions = []string{
+		"s3:GetBucketLocation",
+		"s3:ListBucket",
+		// Add more bucket level read actions here.
+	}
+	readOnlyObjectActions = []string{
+		"s3:GetObject",
+		// Add more object level read actions here.
+	}
+)
+
+// Obtain bucket statement for read-write bucketPolicy.
+func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
+	objectResourceStatement.Effect = "Allow"
+	objectResourceStatement.Principal.AWS = []string{"*"}
+	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
+	objectResourceStatement.Actions = readWriteObjectActions
+	return objectResourceStatement
+}
+
+// Obtain object statement for read-write bucketPolicy.
+func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatement {
+	bucketResourceStatement := policyStatement{}
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal.AWS = []string{"*"}
+	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
+	bucketResourceStatement.Actions = readWriteBucketActions
+	return bucketResourceStatement
+}
+
+// Obtain statements for read-write bucketPolicy.
+func getReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
+	// Save the read write policy.
+	statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
+	return statements
+}
+
+// Obtain bucket statement for read only bucketPolicy.
+func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
+	bucketResourceStatement := policyStatement{}
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal.AWS = []string{"*"}
+	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
+	bucketResourceStatement.Actions = readOnlyBucketActions
+	return bucketResourceStatement
+}
+
+// Obtain object statement for read only bucketPolicy.
+func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
+	objectResourceStatement.Effect = "Allow"
+	objectResourceStatement.Principal.AWS = []string{"*"}
+	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
+	objectResourceStatement.Actions = readOnlyObjectActions
+	return objectResourceStatement
+}
+
+// Obtain statements for read only bucketPolicy.
+func getReadOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
+	// Save the read only policy.
+	statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
+	return statements
+}
+
+// Obtain bucket statements for write only bucketPolicy.
+func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
+
+	bucketResourceStatement := policyStatement{}
+	bucketResourceStatement.Effect = "Allow"
+	bucketResourceStatement.Principal.AWS = []string{"*"}
+	bucketResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName)}
+	bucketResourceStatement.Actions = writeOnlyBucketActions
+	return bucketResourceStatement
+}
+
+// Obtain object statements for write only bucketPolicy.
+func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
+	objectResourceStatement.Effect = "Allow"
+	objectResourceStatement.Principal.AWS = []string{"*"}
+	objectResourceStatement.Resources = []string{fmt.Sprintf("%s%s", AWSResourcePrefix, bucketName+"/"+objectPrefix+"*")}
+	objectResourceStatement.Actions = writeOnlyObjectActions
+	return objectResourceStatement
+}
+
+// Obtain statements for write only bucketPolicy.
+func getWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
+	// Write only policy.
+	// Save the write only policy.
+	statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
+	return statements
+}
+
+// Tests validate Action validator.
+func TestIsValidActions(t *testing.T) {
+	testCases := []struct {
+		// input.
+		actions []string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Action.
+		// Test case - 1.
+		// "s3:ListObject" is an invalid Action.
+		{[]string{"s3:GetObject", "s3:ListObject", "s3:RemoveObject"}, errors.New("Unsupported action found: ‘s3:ListObject’, please validate your policy document."), false},
+		// Test case - 2.
+		// Empty Actions.
+		{[]string{}, errors.New("Action list cannot be empty."), false},
+		// Test case - 3.
+		// "s3:DeleteEverything"" is an invalid Action.
+		{[]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"},
+			errors.New("Unsupported action found: ‘s3:DeleteEverything’, please validate your policy document."), false},
+
+		// Inputs with valid Action.
+		// Test Case - 4.
+		{[]string{"s3:*", "*", "s3:GetObject", "s3:ListBucket",
+			"s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidActions(testCase.actions)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+
+	}
+}
+
+// Tests validate Effect validator.
+func TestIsValidEffect(t *testing.T) {
+	testCases := []struct {
+		// input.
+		effect string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Effect.
+		// Test case - 1.
+		{"DontAllow", errors.New("Unsupported Effect found: ‘DontAllow’, please validate your policy document."), false},
+		// Test case - 2.
+		{"NeverAllow", errors.New("Unsupported Effect found: ‘NeverAllow’, please validate your policy document."), false},
+		// Test case - 3.
+		{"AllowAlways", errors.New("Unsupported Effect found: ‘AllowAlways’, please validate your policy document."), false},
+
+		// Inputs with valid Effect.
+		// Test Case - 4.
+		{"Allow", nil, true},
+		// Test Case - 5.
+		{"Deny", nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidEffect(testCase.effect)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+	}
+}
+
+// Tests validate Resources validator.
+func TestIsValidResources(t *testing.T) {
+	testCases := []struct {
+		// input.
+		resources []string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Action.
+		// Test case - 1.
+		// Empty Resources.
+		{[]string{}, errors.New("Resource list cannot be empty."), false},
+		// Test case - 2.
+		// A valid resource should have prefix "arn:aws:s3:::".
+		{[]string{"my-resource"}, errors.New("Unsupported resource style found: ‘my-resource’, please validate your policy document."), false},
+		// Test case - 3.
+		// A Valid resource should have bucket name followed by "arn:aws:s3:::".
+		{[]string{"arn:aws:s3:::"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document."), false},
+		// Test Case - 4.
+		// Valid resource shouldn't have slash('/') followed by "arn:aws:s3:::".
+		{[]string{"arn:aws:s3:::/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document."), false},
+
+		// Test cases with valid Resources.
+		{[]string{"arn:aws:s3:::my-bucket"}, nil, true},
+		{[]string{"arn:aws:s3:::my-bucket/Asia/*"}, nil, true},
+		{[]string{"arn:aws:s3:::my-bucket/Asia/India/*"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidResources(testCase.resources)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+	}
+}
+
+// Tests validate principals validator.
+func TestIsValidPrincipals(t *testing.T) {
+	testCases := []struct {
+		// input.
+		principals []string
+		// expected output.
+		err error
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Inputs with unsupported Principals.
+		// Test case - 1.
+		// Empty Principals list.
+		{[]string{}, errors.New("Principal cannot be empty."), false},
+		// Test case - 2.
+		// "*" is the only valid principal.
+		{[]string{"my-principal"}, errors.New("Unsupported principal style found: ‘my-principal’, please validate your policy document."), false},
+		// Test case - 3.
+		{[]string{"*", "111122233"}, errors.New("Unsupported principal style found: ‘111122233’, please validate your policy document."), false},
+
+		// Test case - 4.
+		// Test case with valid principal value.
+		{[]string{"*"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		err := isValidPrincipals(testCase.principals)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+	}
+}
+
+// Tests validate policyStatement condition validator.
+func TestIsValidConditions(t *testing.T) {
+	// returns empty conditions map.
+	setEmptyConditions := func() map[string]map[string]string {
+		return make(map[string]map[string]string)
+	}
+
+	// returns map with the "StringEquals" set to empty map.
+	setEmptyStringEquals := func() map[string]map[string]string {
+		emptyMap := make(map[string]string)
+		conditions := make(map[string]map[string]string)
+		conditions["StringEquals"] = emptyMap
+		return conditions
+
+	}
+
+	// returns map with the "StringNotEquals" set to empty map.
+	setEmptyStringNotEquals := func() map[string]map[string]string {
+		emptyMap := make(map[string]string)
+		conditions := make(map[string]map[string]string)
+		conditions["StringNotEquals"] = emptyMap
+		return conditions
+
+	}
+	// Generate conditions.
+	generateConditions := func(key1, key2, value string) map[string]map[string]string {
+		innerMap := make(map[string]string)
+		innerMap[key2] = value
+		conditions := make(map[string]map[string]string)
+		conditions[key1] = innerMap
+		return conditions
+	}
+
+	// generate ambigious conditions.
+	generateAmbigiousConditions := func() map[string]map[string]string {
+		innerMap := make(map[string]string)
+		innerMap["s3:prefix"] = "Asia/"
+		conditions := make(map[string]map[string]string)
+		conditions["StringEquals"] = innerMap
+		conditions["StringNotEquals"] = innerMap
+		return conditions
+	}
+
+	// generate valid and non valid type in the condition map.
+	generateValidInvalidConditions := func() map[string]map[string]string {
+		innerMap := make(map[string]string)
+		innerMap["s3:prefix"] = "Asia/"
+		conditions := make(map[string]map[string]string)
+		conditions["StringEquals"] = innerMap
+		conditions["InvalidType"] = innerMap
+		return conditions
+	}
+
+	// generate valid and invalid keys for valid types in the same condition map.
+	generateValidInvalidConditionKeys := func() map[string]map[string]string {
+		innerMapValid := make(map[string]string)
+		innerMapValid["s3:prefix"] = "Asia/"
+		innerMapInValid := make(map[string]string)
+		innerMapInValid["s3:invalid"] = "Asia/"
+		conditions := make(map[string]map[string]string)
+		conditions["StringEquals"] = innerMapValid
+		conditions["StringEquals"] = innerMapInValid
+		return conditions
+	}
+
+	// List of Conditions used for test cases.
+	testConditions := []map[string]map[string]string{
+		generateConditions("StringValues", "s3:max-keys", "100"),
+		generateConditions("StringEquals", "s3:Object", "100"),
+		generateAmbigiousConditions(),
+		generateValidInvalidConditions(),
+		generateValidInvalidConditionKeys(),
+		setEmptyConditions(),
+		setEmptyStringEquals(),
+		setEmptyStringNotEquals(),
+		generateConditions("StringEquals", "s3:prefix", "Asia/"),
+		generateConditions("StringEquals", "s3:max-keys", "100"),
+		generateConditions("StringNotEquals", "s3:prefix", "Asia/"),
+		generateConditions("StringNotEquals", "s3:max-keys", "100"),
+	}
+
+	testCases := []struct {
+		inputCondition map[string]map[string]string
+		// expected result.
+		expectedErr error
+		// flag indicating whether test should pass.
+		shouldPass bool
+	}{
+		// Malformed conditions.
+		// Test case - 1.
+		// "StringValues" is an invalid type.
+		{testConditions[0], fmt.Errorf("Unsupported condition type 'StringValues', " +
+			"please validate your policy document."), false},
+		// Test case - 2.
+		// "s3:Object" is an invalid key.
+		{testConditions[1], fmt.Errorf("Unsupported condition key " +
+			"'StringEquals', please validate your policy document."), false},
+		// Test case - 3.
+		// Test case with Ambigious conditions set.
+		{testConditions[2], fmt.Errorf("Ambigious condition values for key 's3:prefix', " +
+			"please validate your policy document."), false},
+		// Test case - 4.
+		// Test case with valid and invalid condition types.
+		{testConditions[3], fmt.Errorf("Unsupported condition type 'InvalidType', " +
+			"please validate your policy document."), false},
+		// Test case - 5.
+		// Test case with valid and invalid condition keys.
+		{testConditions[4], fmt.Errorf("Unsupported condition key 'StringEquals', " +
+			"please validate your policy document."), false},
+		// Test cases with valid conditions.
+		// Test case - 6.
+		{testConditions[5], nil, true},
+		// Test case - 7.
+		{testConditions[6], nil, true},
+		// Test case - 8.
+		{testConditions[7], nil, true},
+		// Test case - 9.
+		{testConditions[8], nil, true},
+		// Test case - 10.
+		{testConditions[9], nil, true},
+		// Test case - 11.
+		{testConditions[10], nil, true},
+		// Test case 10.
+		{testConditions[11], nil, true},
+	}
+	for i, testCase := range testCases {
+		actualErr := isValidConditions(testCase.inputCondition)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, actualErr.Error())
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.expectedErr.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if actualErr.Error() != testCase.expectedErr.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.expectedErr.Error(), actualErr.Error())
+			}
+		}
+	}
+}
+
+// Tests validate Policy Action and Resource fields.
+func TestCheckbucketPolicyResources(t *testing.T) {
+	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
+	setValidPrefixActions := func(statements []policyStatement) []policyStatement {
+		statements[0].Actions = []string{"s3:DeleteObject", "s3:PutObject"}
+		return statements
+	}
+	// contracting policy statement with recursive resources.
+	// should result in ErrMalformedPolicy
+	setRecurseResource := func(statements []policyStatement) []policyStatement {
+		statements[0].Resources = []string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}
+		return statements
+	}
+
+	// constructing policy statement with lexically close characters.
+	// should not result in ErrMalformedPolicy
+	setResourceLexical := func(statements []policyStatement) []policyStatement {
+		statements[0].Resources = []string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}
+		return statements
+	}
+
+	// List of bucketPolicy used for tests.
+	bucketAccessPolicies := []bucketPolicy{
+		// bucketPolicy - 1.
+		// Contains valid read only policy statement.
+		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
+		// bucketPolicy - 2.
+		// Contains valid read-write only policy statement.
+		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
+		// bucketPolicy - 3.
+		// Contains valid write only policy statement.
+		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
+		// bucketPolicy - 4.
+		// Contains invalidPrefixActions.
+		// Since resourcePrefix is not to the bucket-name, it return ErrMalformedPolicy.
+		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket-fail", "Asia/India/")},
+		// bucketPolicy - 5.
+		// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
+		// but bucket part of the resource is not equal to the bucket name.
+		// this results in return of ErrMalformedPolicy.
+		{Version: "1.0", Statements: setValidPrefixActions(getWriteOnlyStatement("minio-bucket-fail", "Asia/India/"))},
+		// bucketPolicy - 6.
+		// contracting policy statement with recursive resources.
+		// should result in ErrMalformedPolicy
+		{Version: "1.0", Statements: setRecurseResource(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "")))},
+		// BucketPolciy - 7.
+		// constructing policy statement with non recursive but
+		// lexically close resources.
+		// should result in ErrNone.
+		{Version: "1.0", Statements: setResourceLexical(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "oo")))},
+	}
+
+	testCases := []struct {
+		inputPolicy bucketPolicy
+		// expected results.
+		apiErrCode APIErrorCode
+		// Flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Test case - 1.
+		{bucketAccessPolicies[0], ErrNone, true},
+		// Test case - 2.
+		{bucketAccessPolicies[1], ErrNone, true},
+		// Test case - 3.
+		{bucketAccessPolicies[2], ErrNone, true},
+		// Test case - 4.
+		// contains invalidPrefixActions (check bucket-policy-parser.go).
+		// Resource prefix will not be equal to the bucket name in this case.
+		{bucketAccessPolicies[3], ErrMalformedPolicy, false},
+		// Test case - 5.
+		// actions contain invalidPrefixActions (check bucket-policy-parser.go).
+		// Resource prefix bucket part is not equal to the bucket name in this case.
+		{bucketAccessPolicies[4], ErrMalformedPolicy, false},
+		// Test case - 6.
+		// contracting policy statement with recursive resources.
+		// should result in ErrPolicyNesting.
+		{bucketAccessPolicies[5], ErrPolicyNesting, false},
+		// Test case - 7.
+		// constructing policy statement with lexically close
+		// characters.
+		// should result in ErrNone.
+		{bucketAccessPolicies[6], ErrNone, true},
+	}
+	for i, testCase := range testCases {
+		apiErrCode := checkBucketPolicyResources("minio-bucket", &testCase.inputPolicy)
+		if apiErrCode != ErrNone && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
+		}
+		if apiErrCode == ErrNone && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with ErrCode %v, but passed instead", i+1, testCase.apiErrCode)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if apiErrCode != ErrNone && !testCase.shouldPass {
+			if testCase.apiErrCode != apiErrCode {
+				t.Errorf("Test %d: Expected to fail with error code %v, but instead failed with error code %v", i+1, testCase.apiErrCode, apiErrCode)
+			}
+		}
+	}
+}
+
+// Tests validate parsing of BucketAccessPolicy.
+func TestParseBucketPolicy(t *testing.T) {
+	// set Unsupported Actions.
+	setUnsupportedActions := func(statements []policyStatement) []policyStatement {
+		// "s3:DeleteEverything"" is an Unsupported Action.
+		statements[0].Actions = []string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}
+		return statements
+	}
+	// set unsupported Effect.
+	setUnsupportedEffect := func(statements []policyStatement) []policyStatement {
+		// Effect "Don't allow" is Unsupported.
+		statements[0].Effect = "DontAllow"
+		return statements
+	}
+	// set unsupported principals.
+	setUnsupportedPrincipals := func(statements []policyStatement) []policyStatement {
+		// "User1111"" is an Unsupported Principal.
+		statements[0].Principal.AWS = []string{"*", "User1111"}
+		return statements
+	}
+	// set unsupported Resources.
+	setUnsupportedResources := func(statements []policyStatement) []policyStatement {
+		// "s3:DeleteEverything"" is an Unsupported Action.
+		statements[0].Resources = []string{"my-resource"}
+		return statements
+	}
+	// List of bucketPolicy used for test cases.
+	bucketAccesPolicies := []bucketPolicy{
+		// bucketPolicy - 0.
+		// bucketPolicy statement empty.
+		{Version: "1.0"},
+		// bucketPolicy - 1.
+		// bucketPolicy version empty.
+		{Version: "", Statements: []policyStatement{}},
+		// bucketPolicy - 2.
+		// Readonly bucketPolicy.
+		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
+		// bucketPolicy - 3.
+		// Read-Write bucket policy.
+		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
+		// bucketPolicy - 4.
+		// Write only bucket policy.
+		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
+		// bucketPolicy - 5.
+		// bucketPolicy statement contains unsupported action.
+		{Version: "1.0", Statements: setUnsupportedActions(getReadOnlyStatement("minio-bucket", ""))},
+		// bucketPolicy - 6.
+		// bucketPolicy statement contains unsupported Effect.
+		{Version: "1.0", Statements: setUnsupportedEffect(getReadWriteStatement("minio-bucket", "Asia/"))},
+		// bucketPolicy - 7.
+		// bucketPolicy statement contains unsupported Principal.
+		{Version: "1.0", Statements: setUnsupportedPrincipals(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
+		// bucketPolicy - 8.
+		// bucketPolicy statement contains unsupported Resource.
+		{Version: "1.0", Statements: setUnsupportedResources(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
+	}
+
+	testCases := []struct {
+		inputPolicy bucketPolicy
+		// expected results.
+		expectedPolicy bucketPolicy
+		err            error
+		// Flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		// Test case - 1.
+		// bucketPolicy statement empty.
+		{bucketAccesPolicies[0], bucketPolicy{}, errors.New("Policy statement cannot be empty."), false},
+		// Test case - 2.
+		// bucketPolicy version empty.
+		{bucketAccesPolicies[1], bucketPolicy{}, errors.New("Policy version cannot be empty."), false},
+		// Test case - 3.
+		// Readonly bucketPolicy.
+		{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
+		// Test case - 4.
+		// Read-Write bucket policy.
+		{bucketAccesPolicies[3], bucketAccesPolicies[3], nil, true},
+		// Test case - 5.
+		// Write only bucket policy.
+		{bucketAccesPolicies[4], bucketAccesPolicies[4], nil, true},
+		// Test case - 6.
+		// bucketPolicy statement contains unsupported action.
+		{bucketAccesPolicies[5], bucketAccesPolicies[5], fmt.Errorf("Unsupported action found: ‘s3:DeleteEverything’, please validate your policy document."), false},
+		// Test case - 7.
+		// bucketPolicy statement contains unsupported Effect.
+		{bucketAccesPolicies[6], bucketAccesPolicies[6], fmt.Errorf("Unsupported Effect found: ‘DontAllow’, please validate your policy document."), false},
+		// Test case - 8.
+		// bucketPolicy statement contains unsupported Principal.
+		{bucketAccesPolicies[7], bucketAccesPolicies[7], fmt.Errorf("Unsupported principal style found: ‘User1111’, please validate your policy document."), false},
+		// Test case - 9.
+		// bucketPolicy statement contains unsupported Resource.
+		{bucketAccesPolicies[8], bucketAccesPolicies[8], fmt.Errorf("Unsupported resource style found: ‘my-resource’, please validate your policy document."), false},
+	}
+	for i, testCase := range testCases {
+		var buffer bytes.Buffer
+		encoder := json.NewEncoder(&buffer)
+		err := encoder.Encode(testCase.inputPolicy)
+		if err != nil {
+			t.Fatalf("Test %d: Couldn't Marshal bucket policy %s", i+1, err)
+		}
+
+		var actualAccessPolicy = &bucketPolicy{}
+		err = parseBucketPolicy(&buffer, actualAccessPolicy)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if err.Error() != testCase.err.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
+			}
+		}
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.expectedPolicy.String() != actualAccessPolicy.String() {
+				t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
+			}
+		}
+	}
+}

cmd/bucket-policy.go

@@ -0,0 +1,180 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"path"
+	"sync"
+)
+
+// Variable represents bucket policies in memory.
+var globalBucketPolicies *bucketPolicies
+
+// Global bucket policies list, policies are enforced on each bucket looking
+// through the policies here.
+type bucketPolicies struct {
+	rwMutex *sync.RWMutex
+
+	// Collection of 'bucket' policies.
+	bucketPolicyConfigs map[string]*bucketPolicy
+}
+
+// Fetch bucket policy for a given bucket.
+func (bp bucketPolicies) GetBucketPolicy(bucket string) *bucketPolicy {
+	bp.rwMutex.RLock()
+	defer bp.rwMutex.RUnlock()
+	return bp.bucketPolicyConfigs[bucket]
+}
+
+// Set a new bucket policy for a bucket, this operation will overwrite
+// any previous bucketpolicies for the bucket.
+func (bp *bucketPolicies) SetBucketPolicy(bucket string, policy *bucketPolicy) error {
+	bp.rwMutex.Lock()
+	defer bp.rwMutex.Unlock()
+	if policy == nil {
+		return errors.New("invalid argument")
+	}
+	bp.bucketPolicyConfigs[bucket] = policy
+	return nil
+}
+
+// Remove bucket policy for a bucket, from in-memory map.
+func (bp *bucketPolicies) RemoveBucketPolicy(bucket string) {
+	bp.rwMutex.Lock()
+	defer bp.rwMutex.Unlock()
+	delete(bp.bucketPolicyConfigs, bucket)
+}
+
+// Loads all bucket policies from persistent layer.
+func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolicy, err error) {
+	// List buckets to proceed loading all notification configuration.
+	buckets, err := objAPI.ListBuckets()
+	if err != nil {
+		return nil, err
+	}
+	policies = make(map[string]*bucketPolicy)
+	// Loads bucket policy.
+	for _, bucket := range buckets {
+		var policy *bucketPolicy
+		policy, err = readBucketPolicy(bucket.Name, objAPI)
+		if err != nil {
+			switch err.(type) {
+			case BucketPolicyNotFound:
+				continue
+			}
+			return nil, err
+		}
+		policies[bucket.Name] = policy
+	}
+
+	// Success.
+	return policies, nil
+
+}
+
+// Intialize all bucket policies.
+func initBucketPolicies(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+	// Read all bucket policies.
+	policies, err := loadAllBucketPolicies(objAPI)
+	if err != nil {
+		return err
+	}
+
+	globalBucketPolicies = &bucketPolicies{
+		rwMutex:             &sync.RWMutex{},
+		bucketPolicyConfigs: policies,
+	}
+	return nil
+}
+
+// getOldBucketsConfigPath - get old buckets config path. (Only used for migrating old bucket policies)
+func getOldBucketsConfigPath() (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	return path.Join(configPath, "buckets"), nil
+}
+
+// readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
+// if bucket policy is not found. This function also parses the bucket policy into an object.
+func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
+	// Verify bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return nil, BucketNameInvalid{Bucket: bucket}
+	}
+	policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
+	objInfo, err := objAPI.GetObjectInfo(minioMetaBucket, policyPath)
+	if err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			return nil, BucketPolicyNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	var buffer bytes.Buffer
+	err = objAPI.GetObject(minioMetaBucket, policyPath, 0, objInfo.Size, &buffer)
+	if err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			return nil, BucketPolicyNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	// Parse the saved policy.
+	var policy = &bucketPolicy{}
+	err = parseBucketPolicy(&buffer, policy)
+	if err != nil {
+		return nil, err
+
+	}
+	return policy, nil
+}
+
+// removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound
+// if no policies are found.
+func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
+	// Verify bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+
+	policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
+	if err := objAPI.DeleteObject(minioMetaBucket, policyPath); err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			return BucketPolicyNotFound{Bucket: bucket}
+		}
+		return err
+	}
+	return nil
+}
+
+// writeBucketPolicy - save all bucket policies.
+func writeBucketPolicy(bucket string, objAPI ObjectLayer, reader io.Reader, size int64) error {
+	// Verify if bucket path legal
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+
+	policyPath := pathJoin(bucketConfigPrefix, bucket, policyJSON)
+	_, err := objAPI.PutObject(minioMetaBucket, policyPath, size, reader, nil)
+	return err
+}

cmd/build-constants.go

@@ -0,0 +1,33 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
+// set through ‘buildscripts/gen-ldflags.go’.
+var (
+	// GOPATH - GOPATH value at the time of build.
+	GOPATH = ""
+
+	// Version - version time.RFC3339.
+	Version = "DEVELOPMENT.GOGET"
+	// ReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ.
+	ReleaseTag = "DEVELOPMENT.GOGET"
+	// CommitID - latest commit id.
+	CommitID = "DEVELOPMENT.GOGET"
+	// ShortCommitID - first 12 characters from CommitID.
+	ShortCommitID = CommitID[:12]
+)

cmd/certs.go

@@ -0,0 +1,89 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// createCertsPath create certs path.
+func createCertsPath() error {
+	certsPath, err := getCertsPath()
+	if err != nil {
+		return err
+	}
+	return os.MkdirAll(certsPath, 0700)
+}
+
+// getCertsPath get certs path.
+func getCertsPath() (string, error) {
+	var certsPath string
+	configDir, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	certsPath = filepath.Join(configDir, globalMinioCertsDir)
+	return certsPath, nil
+}
+
+// mustGetCertsPath must get certs path.
+func mustGetCertsPath() string {
+	certsPath, err := getCertsPath()
+	fatalIf(err, "Failed to get certificate path.")
+	return certsPath
+}
+
+// mustGetCertFile must get cert file.
+func mustGetCertFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioCertFile)
+}
+
+// mustGetKeyFile must get key file.
+func mustGetKeyFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioKeyFile)
+}
+
+// isCertFileExists verifies if cert file exists, returns true if
+// found, false otherwise.
+func isCertFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioCertFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isKeyFileExists verifies if key file exists, returns true if found,
+// false otherwise.
+func isKeyFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioKeyFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isSSL - returns true with both cert and key exists.
+func isSSL() bool {
+	if isCertFileExists() && isKeyFileExists() {
+		return true
+	}
+	return false
+}

cmd/certs_test.go

@@ -0,0 +1,60 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// Make sure we have a valid certs path.
+func TestGetCertsPath(t *testing.T) {
+	path, err := getCertsPath()
+	if err != nil {
+		t.Error(err)
+	}
+	if path == "" {
+		t.Errorf("expected path to not be an empty string, got: '%s'", path)
+	}
+	// Ensure it contains some sort of path separator.
+	if !strings.ContainsRune(path, os.PathSeparator) {
+		t.Errorf("expected path to contain file separator")
+	}
+	// It should also be an absolute path.
+	if !filepath.IsAbs(path) {
+		t.Errorf("expected path to be an absolute path")
+	}
+
+	// This will error if something goes wrong, so just call it.
+	mustGetCertsPath()
+}
+
+// Ensure that the certificate and key file getters contain their respective
+// file name and endings.
+func TestGetFiles(t *testing.T) {
+	file := mustGetCertFile()
+	if !strings.Contains(file, globalMinioCertFile) {
+		t.Errorf("CertFile does not contain %s", globalMinioCertFile)
+	}
+
+	file = mustGetKeyFile()
+	if !strings.Contains(file, globalMinioKeyFile) {
+		t.Errorf("KeyFile does not contain %s", globalMinioKeyFile)
+	}
+}

cmd/checkport.go

@@ -0,0 +1,48 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net"
+)
+
+// Make sure that none of the other processes are listening on the
+// specified port on any of the interfaces.
+//
+// On linux if a process is listening on 127.0.0.1:9000 then Listen()
+// on ":9000" fails with the error "port already in use".
+// However on Mac OSX Listen() on ":9000" falls back to the IPv6 address.
+// This causes confusion on Mac OSX that minio server is not reachable
+// on 127.0.0.1 even though minio server is running. So before we start
+// the minio server we make sure that the port is free on each tcp network.
+func checkPortAvailability(port int) error {
+	network := [3]string{"tcp", "tcp4", "tcp6"}
+	for _, n := range network {
+		l, err := net.Listen(n, fmt.Sprintf(":%d", port))
+		if err != nil {
+			return err
+		}
+
+		// look for error so we don't have dangling connection
+		if err = l.Close(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/checkport_test.go

@@ -0,0 +1,55 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"runtime"
+	"testing"
+)
+
+// Tests for port availability logic written for server startup sequence.
+func TestCheckPortAvailability(t *testing.T) {
+	tests := []struct {
+		port int
+	}{
+		{getFreePort()},
+		{getFreePort()},
+	}
+	for _, test := range tests {
+		// This test should pass if the ports are available
+		err := checkPortAvailability(test.port)
+		if err != nil {
+			t.Fatalf("checkPortAvailability test failed for port: %d. Error: %v", test.port, err)
+		}
+
+		// Now use the ports and check again
+		ln, err := net.Listen("tcp", fmt.Sprintf(":%d", test.port))
+		if err != nil {
+			t.Fail()
+		}
+		defer ln.Close()
+
+		err = checkPortAvailability(test.port)
+
+		// Skip if the os is windows due to https://github.com/golang/go/issues/7598
+		if err == nil && runtime.GOOS != "windows" {
+			t.Fatalf("checkPortAvailability should fail for port: %d. Error: %v", test.port, err)
+		}
+	}
+}

cmd/commands.go

@@ -0,0 +1,44 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "github.com/minio/cli"
+
+// Collection of minio commands currently supported are.
+var commands = []cli.Command{}
+
+// Collection of minio commands currently supported in a trie tree.
+var commandsTree = newTrie()
+
+// Collection of minio flags currently supported.
+var globalFlags = []cli.Flag{
+	cli.StringFlag{
+		Name:  "config-dir, C",
+		Value: mustGetConfigPath(),
+		Usage: "Path to configuration folder.",
+	},
+	cli.BoolFlag{
+		Name:  "quiet",
+		Usage: "Suppress chatty output.",
+	},
+}
+
+// registerCommand registers a cli command.
+func registerCommand(command cli.Command) {
+	commands = append(commands, command)
+	commandsTree.Insert(command.Name)
+}

cmd/config-migrate.go

@@ -0,0 +1,258 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/quick"
+)
+
+func migrateConfig() {
+	// Purge all configs with version '1'.
+	purgeV1()
+	// Migrate version '2' to '3'.
+	migrateV2ToV3()
+	// Migrate version '3' to '4'.
+	migrateV3ToV4()
+	// Migrate version '4' to '5'.
+	migrateV4ToV5()
+	// Migrate version '5' to '6.
+	migrateV5ToV6()
+}
+
+// Version '1' is not supported anymore and deprecated, safe to delete.
+func purgeV1() {
+	cv1, err := loadConfigV1()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘1’.")
+
+	if cv1.Version == "1" {
+		console.Println("Removed unsupported config version ‘1’.")
+		/// Purge old fsUsers.json file
+		configPath, err := getConfigPath()
+		fatalIf(err, "Unable to retrieve config path.")
+
+		configFile := filepath.Join(configPath, "fsUsers.json")
+		removeAll(configFile)
+	}
+	fatalIf(errors.New(""), "Failed to migrate unrecognized config version ‘"+cv1.Version+"’.")
+}
+
+// Version '2' to '3' config migration adds new fields and re-orders
+// previous fields. Simplifies config for future additions.
+func migrateV2ToV3() {
+	cv2, err := loadConfigV2()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘2’.")
+	if cv2.Version != "2" {
+		return
+	}
+	srvConfig := &configV3{}
+	srvConfig.Version = "3"
+	srvConfig.Addr = ":9000"
+	srvConfig.Credential = credential{
+		AccessKeyID:     cv2.Credentials.AccessKeyID,
+		SecretAccessKey: cv2.Credentials.SecretAccessKey,
+	}
+	srvConfig.Region = cv2.Credentials.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature V4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = consoleLogger{
+		Enable: true,
+		Level:  "fatal",
+	}
+	flogger := fileLogger{}
+	flogger.Level = "error"
+	if cv2.FileLogger.Filename != "" {
+		flogger.Enable = true
+		flogger.Filename = cv2.FileLogger.Filename
+	}
+	srvConfig.Logger.File = flogger
+
+	slogger := syslogLogger{}
+	slogger.Level = "debug"
+	if cv2.SyslogLogger.Addr != "" {
+		slogger.Enable = true
+		slogger.Addr = cv2.SyslogLogger.Addr
+	}
+	srvConfig.Logger.Syslog = slogger
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err, "Unable to initialize config.")
+
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	// Migrate the config.
+	err = qc.Save(configFile)
+	fatalIf(err, "Failed to migrate config from ‘"+cv2.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+
+	console.Println("Migration from version ‘" + cv2.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}
+
+// Version '3' to '4' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV3ToV4() {
+	cv3, err := loadConfigV3()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘3’.")
+	if cv3.Version != "3" {
+		return
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &configV4{}
+	srvConfig.Version = "4"
+	srvConfig.Credential = cv3.Credential
+	srvConfig.Region = cv3.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv3.Logger.Console
+	srvConfig.Logger.File = cv3.Logger.File
+	srvConfig.Logger.Syslog = cv3.Logger.Syslog
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err, "Unable to initialize the quick config.")
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	err = qc.Save(configFile)
+	fatalIf(err, "Failed to migrate config from ‘"+cv3.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+
+	console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}
+
+// Version '5' to '6' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV5ToV6() {
+	cv5, err := loadConfigV5()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘5’.")
+	if cv5.Version != "5" {
+		return
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &serverConfigV6{}
+	srvConfig.Version = globalMinioConfigVersion
+	srvConfig.Credential = cv5.Credential
+	srvConfig.Region = cv5.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv5.Logger.Console
+	srvConfig.Logger.File = cv5.Logger.File
+	srvConfig.Logger.Syslog = cv5.Logger.Syslog
+	srvConfig.Notify.AMQP = map[string]amqpNotify{
+		"1": {
+			Enable:      cv5.Logger.AMQP.Enable,
+			URL:         cv5.Logger.AMQP.URL,
+			Exchange:    cv5.Logger.AMQP.Exchange,
+			RoutingKey:  cv5.Logger.AMQP.RoutingKey,
+			Mandatory:   cv5.Logger.AMQP.Mandatory,
+			Immediate:   cv5.Logger.AMQP.Immediate,
+			Durable:     cv5.Logger.AMQP.Durable,
+			Internal:    cv5.Logger.AMQP.Internal,
+			NoWait:      cv5.Logger.AMQP.NoWait,
+			AutoDeleted: cv5.Logger.AMQP.AutoDeleted,
+		},
+	}
+	srvConfig.Notify.ElasticSearch = map[string]elasticSearchNotify{
+		"1": {
+			Enable: cv5.Logger.ElasticSearch.Enable,
+			URL:    cv5.Logger.ElasticSearch.URL,
+			Index:  cv5.Logger.ElasticSearch.Index,
+		},
+	}
+	srvConfig.Notify.Redis = map[string]redisNotify{
+		"1": {
+			Enable:   cv5.Logger.Redis.Enable,
+			Addr:     cv5.Logger.Redis.Addr,
+			Password: cv5.Logger.Redis.Password,
+			Key:      cv5.Logger.Redis.Key,
+		},
+	}
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err, "Unable to initialize the quick config.")
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	err = qc.Save(configFile)
+	fatalIf(err, "Failed to migrate config from ‘"+cv5.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+
+	console.Println("Migration from version ‘" + cv5.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}
+
+// Version '4' to '5' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV4ToV5() {
+	cv4, err := loadConfigV4()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘4’.")
+	if cv4.Version != "4" {
+		return
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &configV5{}
+	srvConfig.Version = globalMinioConfigVersion
+	srvConfig.Credential = cv4.Credential
+	srvConfig.Region = cv4.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv4.Logger.Console
+	srvConfig.Logger.File = cv4.Logger.File
+	srvConfig.Logger.Syslog = cv4.Logger.Syslog
+	srvConfig.Logger.AMQP.Enable = false
+	srvConfig.Logger.ElasticSearch.Enable = false
+	srvConfig.Logger.Redis.Enable = false
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err, "Unable to initialize the quick config.")
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	err = qc.Save(configFile)
+	fatalIf(err, "Failed to migrate config from ‘"+cv4.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+
+	console.Println("Migration from version ‘" + cv4.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}

cmd/config-old.go

@@ -0,0 +1,276 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/minio/pkg/quick"
+)
+
+/////////////////// Config V1 ///////////////////
+type configV1 struct {
+	Version         string `json:"version"`
+	AccessKeyID     string `json:"accessKeyId"`
+	SecretAccessKey string `json:"secretAccessKey"`
+}
+
+// loadConfigV1 load config
+func loadConfigV1() (*configV1, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return nil, err
+	}
+	configFile := filepath.Join(configPath, "fsUsers.json")
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV1{}
+	c.Version = "1"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+/////////////////// Config V2 ///////////////////
+type configV2 struct {
+	Version     string `json:"version"`
+	Credentials struct {
+		AccessKeyID     string `json:"accessKeyId"`
+		SecretAccessKey string `json:"secretAccessKey"`
+		Region          string `json:"region"`
+	} `json:"credentials"`
+	MongoLogger struct {
+		Addr       string `json:"addr"`
+		DB         string `json:"db"`
+		Collection string `json:"collection"`
+	} `json:"mongoLogger"`
+	SyslogLogger struct {
+		Network string `json:"network"`
+		Addr    string `json:"addr"`
+	} `json:"syslogLogger"`
+	FileLogger struct {
+		Filename string `json:"filename"`
+	} `json:"fileLogger"`
+}
+
+// loadConfigV2 load config version '2'.
+func loadConfigV2() (*configV2, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV2{}
+	c.Version = "2"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+/////////////////// Config V3 ///////////////////
+
+// backendV3 type.
+type backendV3 struct {
+	Type  string   `json:"type"`
+	Disk  string   `json:"disk,omitempty"`
+	Disks []string `json:"disks,omitempty"`
+}
+
+// loggerV3 type.
+type loggerV3 struct {
+	Console struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+	}
+	File struct {
+		Enable   bool   `json:"enable"`
+		Filename string `json:"fileName"`
+		Level    string `json:"level"`
+	}
+	Syslog struct {
+		Enable bool   `json:"enable"`
+		Addr   string `json:"address"`
+		Level  string `json:"level"`
+	} `json:"syslog"`
+	// Add new loggers here.
+}
+
+// configV3 server configuration version '3'.
+type configV3 struct {
+	Version string `json:"version"`
+
+	// Backend configuration.
+	Backend backendV3 `json:"backend"`
+
+	// http Server configuration.
+	Addr string `json:"address"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV3 `json:"logger"`
+}
+
+// loadConfigV3 load config version '3'.
+func loadConfigV3() (*configV3, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV3{}
+	c.Version = "3"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// logger type representing version '4' logger config.
+type loggerV4 struct {
+	Console struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+	} `json:"console"`
+	File struct {
+		Enable   bool   `json:"enable"`
+		Filename string `json:"fileName"`
+		Level    string `json:"level"`
+	} `json:"file"`
+	Syslog struct {
+		Enable bool   `json:"enable"`
+		Addr   string `json:"address"`
+		Level  string `json:"level"`
+	} `json:"syslog"`
+}
+
+// configV4 server configuration version '4'.
+type configV4 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV4 `json:"logger"`
+}
+
+// loadConfigV4 load config version '4'.
+func loadConfigV4() (*configV4, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV4{}
+	c.Version = "4"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// logger type representing version '5' logger config.
+type loggerV5 struct {
+	Console struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+	} `json:"console"`
+	File struct {
+		Enable   bool   `json:"enable"`
+		Filename string `json:"fileName"`
+		Level    string `json:"level"`
+	} `json:"file"`
+	Syslog struct {
+		Enable bool   `json:"enable"`
+		Addr   string `json:"address"`
+		Level  string `json:"level"`
+	} `json:"syslog"`
+	AMQP struct {
+		Enable       bool   `json:"enable"`
+		Level        string `json:"level"`
+		URL          string `json:"url"`
+		Exchange     string `json:"exchange"`
+		RoutingKey   string `json:"routingKey"`
+		ExchangeType string `json:"exchangeType"`
+		Mandatory    bool   `json:"mandatory"`
+		Immediate    bool   `json:"immediate"`
+		Durable      bool   `json:"durable"`
+		Internal     bool   `json:"internal"`
+		NoWait       bool   `json:"noWait"`
+		AutoDeleted  bool   `json:"autoDeleted"`
+	} `json:"amqp"`
+	ElasticSearch struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+		URL    string `json:"url"`
+		Index  string `json:"index"`
+	} `json:"elasticsearch"`
+	Redis struct {
+		Enable   bool   `json:"enable"`
+		Level    string `json:"level"`
+		Addr     string `json:"address"`
+		Password string `json:"password"`
+		Key      string `json:"key"`
+	} `json:"redis"`
+}
+
+// configV5 server configuration version '5'.
+type configV5 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV5 `json:"logger"`
+}
+
+// loadConfigV5 load config version '5'.
+func loadConfigV5() (*configV5, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV5{}
+	c.Version = "5"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

cmd/config-v6.go

@@ -0,0 +1,255 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"sync"
+
+	"github.com/minio/minio/pkg/quick"
+)
+
+// serverConfigV6 server configuration version '5'.
+type serverConfigV6 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger logger `json:"logger"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Read Write mutex.
+	rwMutex *sync.RWMutex
+}
+
+// initConfig - initialize server config. config version (called only once).
+func initConfig() error {
+	if !isConfigFileExists() {
+		// Initialize server config.
+		srvCfg := &serverConfigV6{}
+		srvCfg.Version = globalMinioConfigVersion
+		srvCfg.Region = "us-east-1"
+		srvCfg.Credential = mustGenAccessKeys()
+
+		// Enable console logger by default on a fresh run.
+		srvCfg.Logger.Console = consoleLogger{
+			Enable: true,
+			Level:  "fatal",
+		}
+		srvCfg.rwMutex = &sync.RWMutex{}
+
+		// Create config path.
+		err := createConfigPath()
+		if err != nil {
+			return err
+		}
+
+		// Save the new config globally.
+		serverConfig = srvCfg
+
+		// Save config into file.
+		return serverConfig.Save()
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return err
+	}
+	srvCfg := &serverConfigV6{}
+	srvCfg.Version = globalMinioConfigVersion
+	srvCfg.rwMutex = &sync.RWMutex{}
+	qc, err := quick.New(srvCfg)
+	if err != nil {
+		return err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return err
+	}
+	// Save the loaded config globally.
+	serverConfig = srvCfg
+	// Set the version properly after the unmarshalled json is loaded.
+	serverConfig.Version = globalMinioConfigVersion
+
+	return nil
+}
+
+// serverConfig server config.
+var serverConfig *serverConfigV6
+
+// GetVersion get current config version.
+func (s serverConfigV6) GetVersion() string {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Version
+}
+
+/// Logger related.
+
+func (s *serverConfigV6) SetAMQPNotifyByID(accountID string, amqpn amqpNotify) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Notify.AMQP[accountID] = amqpn
+}
+
+func (s serverConfigV6) GetAMQP() map[string]amqpNotify {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Notify.AMQP
+}
+
+// GetAMQPNotify get current AMQP logger.
+func (s serverConfigV6) GetAMQPNotifyByID(accountID string) amqpNotify {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Notify.AMQP[accountID]
+}
+
+func (s *serverConfigV6) SetElasticSearchNotifyByID(accountID string, esNotify elasticSearchNotify) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Notify.ElasticSearch[accountID] = esNotify
+}
+
+func (s serverConfigV6) GetElasticSearch() map[string]elasticSearchNotify {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Notify.ElasticSearch
+}
+
+// GetElasticSearchNotify get current ElasicSearch logger.
+func (s serverConfigV6) GetElasticSearchNotifyByID(accountID string) elasticSearchNotify {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Notify.ElasticSearch[accountID]
+}
+
+func (s *serverConfigV6) SetRedisNotifyByID(accountID string, rNotify redisNotify) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Notify.Redis[accountID] = rNotify
+}
+
+func (s serverConfigV6) GetRedis() map[string]redisNotify {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Notify.Redis
+}
+
+// GetRedisNotify get current Redis logger.
+func (s serverConfigV6) GetRedisNotifyByID(accountID string) redisNotify {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Notify.Redis[accountID]
+}
+
+// SetFileLogger set new file logger.
+func (s *serverConfigV6) SetFileLogger(flogger fileLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.File = flogger
+}
+
+// GetFileLogger get current file logger.
+func (s serverConfigV6) GetFileLogger() fileLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.File
+}
+
+// SetConsoleLogger set new console logger.
+func (s *serverConfigV6) SetConsoleLogger(clogger consoleLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.Console = clogger
+}
+
+// GetConsoleLogger get current console logger.
+func (s serverConfigV6) GetConsoleLogger() consoleLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.Console
+}
+
+// SetSyslogLogger set new syslog logger.
+func (s *serverConfigV6) SetSyslogLogger(slogger syslogLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.Syslog = slogger
+}
+
+// GetSyslogLogger get current syslog logger.
+func (s *serverConfigV6) GetSyslogLogger() syslogLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.Syslog
+}
+
+// SetRegion set new region.
+func (s *serverConfigV6) SetRegion(region string) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Region = region
+}
+
+// GetRegion get current region.
+func (s serverConfigV6) GetRegion() string {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Region
+}
+
+// SetCredentials set new credentials.
+func (s *serverConfigV6) SetCredential(creds credential) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Credential = creds
+}
+
+// GetCredentials get current credentials.
+func (s serverConfigV6) GetCredential() credential {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Credential
+}
+
+// Save config.
+func (s serverConfigV6) Save() error {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+
+	// get config file.
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err
+	}
+
+	// initialize quick.
+	qc, err := quick.New(&s)
+	if err != nil {
+		return err
+	}
+
+	// Save config file.
+	return qc.Save(configFile)
+}

cmd/config.go

@@ -0,0 +1,88 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/go-homedir"
+)
+
+// configPath for custom config path only for testing purposes
+var customConfigPath string
+
+// Sets a new config path.
+func setGlobalConfigPath(configPath string) {
+	customConfigPath = configPath
+}
+
+// getConfigPath get server config path
+func getConfigPath() (string, error) {
+	if customConfigPath != "" {
+		return customConfigPath, nil
+	}
+	homeDir, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+	configPath := filepath.Join(homeDir, globalMinioConfigDir)
+	return configPath, nil
+}
+
+// mustGetConfigPath must get server config path.
+func mustGetConfigPath() string {
+	configPath, err := getConfigPath()
+	fatalIf(err, "Unable to get config path.")
+	return configPath
+}
+
+// createConfigPath create server config path.
+func createConfigPath() error {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return err
+	}
+	return os.MkdirAll(configPath, 0700)
+}
+
+// isConfigFileExists - returns true if config file exists.
+func isConfigFileExists() bool {
+	st, err := os.Stat(mustGetConfigFile())
+	// If file exists and is regular return true.
+	if err == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// mustGetConfigFile must get server config file.
+func mustGetConfigFile() string {
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	return configFile
+}
+
+// getConfigFile get server config file.
+func getConfigFile() (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(configPath, globalMinioConfigFile), nil
+}

cmd/control-main.go

@@ -0,0 +1,134 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"net/rpc"
+	"net/url"
+
+	"github.com/minio/cli"
+)
+
+// "minio control" command.
+var controlCmd = cli.Command{
+	Name:   "control",
+	Usage:  "Control and manage minio server.",
+	Action: mainControl,
+	Subcommands: []cli.Command{
+		healCmd,
+	},
+}
+
+func mainControl(c *cli.Context) {
+	cli.ShowCommandHelp(c, "")
+}
+
+var healCmd = cli.Command{
+	Name:   "heal",
+	Usage:  "To heal objects.",
+	Action: healControl,
+	CustomHelpTemplate: `NAME:
+  minio control {{.Name}} - {{.Usage}}
+
+USAGE:
+  minio control {{.Name}}
+
+EAMPLES:
+  1. Heal an object.
+     $ minio control {{.Name}} http://localhost:9000/songs/classical/western/piano.mp3
+
+  2. Heal all objects in a bucket recursively.
+     $ minio control {{.Name}}  http://localhost:9000/songs
+
+  3. Heall all objects with a given prefix recursively.
+     $ minio control {{.Name}} http://localhost:9000/songs/classical/
+`,
+}
+
+// "minio control heal" entry point.
+func healControl(c *cli.Context) {
+	// Parse bucket and object from url.URL.Path
+	parseBucketObject := func(path string) (bucketName string, objectName string) {
+		splits := strings.SplitN(path, string(slashSeparator), 3)
+		switch len(splits) {
+		case 0, 1:
+			bucketName = ""
+			objectName = ""
+		case 2:
+			bucketName = splits[1]
+			objectName = ""
+		case 3:
+			bucketName = splits[1]
+			objectName = splits[2]
+
+		}
+		return bucketName, objectName
+	}
+
+	if len(c.Args()) != 1 {
+		cli.ShowCommandHelpAndExit(c, "heal", 1)
+	}
+
+	parsedURL, err := url.ParseRequestURI(c.Args()[0])
+	fatalIf(err, "Unable to parse URL")
+
+	bucketName, objectName := parseBucketObject(parsedURL.Path)
+	if bucketName == "" {
+		cli.ShowCommandHelpAndExit(c, "heal", 1)
+	}
+
+	client, err := rpc.DialHTTPPath("tcp", parsedURL.Host, healPath)
+	fatalIf(err, "Unable to connect to %s", parsedURL.Host)
+
+	// If object does not have trailing "/" then it's an object, hence heal it.
+	if objectName != "" && !strings.HasSuffix(objectName, slashSeparator) {
+		fmt.Printf("Healing : /%s/%s", bucketName, objectName)
+		args := &HealObjectArgs{bucketName, objectName}
+		reply := &HealObjectReply{}
+		err = client.Call("Heal.HealObject", args, reply)
+		fatalIf(err, "RPC Heal.HealObject call failed")
+		fmt.Println()
+		return
+	}
+
+	// Recursively list and heal the objects.
+	prefix := objectName
+	marker := ""
+	for {
+		args := HealListArgs{bucketName, prefix, marker, "", 1000}
+		reply := &HealListReply{}
+		err = client.Call("Heal.ListObjects", args, reply)
+		fatalIf(err, "RPC Heal.ListObjects call failed")
+
+		// Heal the objects returned in the ListObjects reply.
+		for _, obj := range reply.Objects {
+			fmt.Printf("Healing : /%s/%s", bucketName, obj)
+			reply := &HealObjectReply{}
+			err = client.Call("Heal.HealObject", HealObjectArgs{bucketName, obj}, reply)
+			fatalIf(err, "RPC Heal.HealObject call failed")
+			fmt.Println()
+		}
+		if !reply.IsTruncated {
+			// End of listing.
+			break
+		}
+		marker = reply.NextMarker
+	}
+}

cmd/crossdomain-xml-handler.go

@@ -0,0 +1,49 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "net/http"
+
+// Standard cross domain policy information located at https://s3.amazonaws.com/crossdomain.xml
+var crossDomainXML = `<?xml version="1.0"?><!DOCTYPE cross-domain-policy SYSTEM "http://www.adobe.com/xml/dtds/cross-domain-policy.dtd"><cross-domain-policy><allow-access-from domain="*" secure="false" /></cross-domain-policy>`
+
+// Cross domain policy implements http.Handler interface, implementing a custom ServerHTTP.
+type crossDomainPolicy struct {
+	handler http.Handler
+}
+
+// A cross-domain policy file is an XML document that grants a web client, such as Adobe Flash Player
+// or Adobe Acrobat (though not necessarily limited to these), permission to handle data across domains.
+// When clients request content hosted on a particular source domain and that content make requests
+// directed towards a domain other than its own, the remote domain needs to host a cross-domain
+// policy file that grants access to the source domain, allowing the client to continue the transaction.
+func setCrossDomainPolicy(h http.Handler) http.Handler {
+	return crossDomainPolicy{handler: h}
+}
+
+func (c crossDomainPolicy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Look for 'crossdomain.xml' in the incoming request.
+	switch r.URL.Path {
+	case "/crossdomain.xml":
+		// Write the standard cross domain policy xml.
+		w.Write([]byte(crossDomainXML))
+		// Request completed, no need to serve to other handlers.
+		return
+	}
+	// Continue to serve the request further.
+	c.handler.ServeHTTP(w, r)
+}

cmd/damerau-levenshtein.go

@@ -0,0 +1,66 @@
+/*
+ * Minio Client (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"math"
+)
+
+// Returns the minimum value of a slice of integers
+func minimum(integers []int) (minVal int) {
+	minVal = math.MaxInt32
+	for _, v := range integers {
+		if v < minVal {
+			minVal = v
+		}
+	}
+	return
+}
+
+// DamerauLevenshteinDistance calculates distance between two strings using an algorithm
+// described in https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance
+func DamerauLevenshteinDistance(a string, b string) int {
+	d := make([][]int, len(a)+1)
+	for i := 1; i <= len(a)+1; i++ {
+		d[i-1] = make([]int, len(b)+1)
+	}
+	for i := 0; i <= len(a); i++ {
+		d[i][0] = i
+	}
+	for j := 0; j <= len(b); j++ {
+		d[0][j] = j
+	}
+	for i := 1; i <= len(a); i++ {
+		for j := 1; j <= len(b); j++ {
+			cost := 0
+			if a[i-1] == b[j-1] {
+				cost = 0
+			} else {
+				cost = 1
+			}
+			d[i][j] = minimum([]int{
+				d[i-1][j] + 1,
+				d[i][j-1] + 1,
+				d[i-1][j-1] + cost,
+			})
+			if i > 1 && j > 1 && a[i-1] == b[j-2] && a[i-2] == b[j-1] {
+				d[i][j] = minimum([]int{d[i][j], d[i-2][j-2] + cost}) // transposition
+			}
+		}
+	}
+	return d[len(a)][len(b)]
+}

cmd/damerau-levenshtein_test.go

@@ -0,0 +1,66 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"math"
+	"testing"
+)
+
+// Test minimum function which calculates the minimal value in a list of integers
+func TestMinimum(t *testing.T) {
+	type testCase struct {
+		listval  []int
+		expected int
+		pass     bool
+	}
+	testCases := []testCase{
+		{listval: []int{3, 4, 15}, expected: 3},
+		{listval: []int{}, expected: math.MaxInt32},
+	}
+	// Validate all the test cases.
+	for i, tt := range testCases {
+		val := minimum(tt.listval)
+		if val != tt.expected {
+			t.Errorf("Test %d:, Expected %d, got %d", i+1, tt.expected, val)
+		}
+	}
+}
+
+// Test DamerauLevenshtein which calculates the difference distance between two words
+func TestDamerauLevenshtein(t *testing.T) {
+	type testCase struct {
+		word1    string
+		word2    string
+		distance int
+	}
+	testCases := []testCase{
+		{word1: "", word2: "", distance: 0},
+		{word1: "a", word2: "a", distance: 0},
+		{word1: "a", word2: "b", distance: 1},
+		{word1: "rm", word2: "tm", distance: 1},
+		{word1: "version", word2: "evrsion", distance: 1},
+		{word1: "version", word2: "bersio", distance: 2},
+	}
+	// Validate all the test cases.
+	for i, tt := range testCases {
+		d := DamerauLevenshteinDistance(tt.word1, tt.word2)
+		if d != tt.distance {
+			t.Errorf("Test %d:, Expected %d, got %d", i+1, tt.distance, d)
+		}
+	}
+}

cmd/erasure-createfile.go

@@ -0,0 +1,145 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/hex"
+	"hash"
+	"io"
+	"sync"
+
+	"github.com/klauspost/reedsolomon"
+)
+
+// erasureCreateFile - writes an entire stream by erasure coding to
+// all the disks, writes also calculate individual block's checksum
+// for future bit-rot protection.
+func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, blockSize int64, dataBlocks int, parityBlocks int, algo string, writeQuorum int) (bytesWritten int64, checkSums []string, err error) {
+	// Allocated blockSized buffer for reading.
+	buf := make([]byte, blockSize)
+
+	hashWriters := newHashWriters(len(disks), algo)
+
+	// Read until io.EOF, erasure codes data and writes to all disks.
+	for {
+		var blocks [][]byte
+		n, rErr := io.ReadFull(reader, buf)
+		// FIXME: this is a bug in Golang, n == 0 and err ==
+		// io.ErrUnexpectedEOF for io.ReadFull function.
+		if n == 0 && rErr == io.ErrUnexpectedEOF {
+			return 0, nil, rErr
+		}
+		if rErr == io.EOF {
+			// We have reached EOF on the first byte read, io.Reader
+			// must be 0bytes, we don't need to erasure code
+			// data. Will create a 0byte file instead.
+			if bytesWritten == 0 {
+				blocks = make([][]byte, len(disks))
+				rErr = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum)
+				if rErr != nil {
+					return 0, nil, rErr
+				}
+			} // else we have reached EOF after few reads, no need to
+			// add an additional 0bytes at the end.
+			break
+		}
+		if rErr != nil && rErr != io.ErrUnexpectedEOF {
+			return 0, nil, rErr
+		}
+		if n > 0 {
+			// Returns encoded blocks.
+			var enErr error
+			blocks, enErr = encodeData(buf[0:n], dataBlocks, parityBlocks)
+			if enErr != nil {
+				return 0, nil, enErr
+			}
+
+			// Write to all disks.
+			if err = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum); err != nil {
+				return 0, nil, err
+			}
+			bytesWritten += int64(n)
+		}
+	}
+
+	checkSums = make([]string, len(disks))
+	for i := range checkSums {
+		checkSums[i] = hex.EncodeToString(hashWriters[i].Sum(nil))
+	}
+	return bytesWritten, checkSums, nil
+}
+
+// encodeData - encodes incoming data buffer into
+// dataBlocks+parityBlocks returns a 2 dimensional byte array.
+func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) {
+	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
+	if err != nil {
+		return nil, err
+	}
+	// Split the input buffer into data and parity blocks.
+	var blocks [][]byte
+	blocks, err = rs.Split(dataBuffer)
+	if err != nil {
+		return nil, err
+	}
+
+	// Encode parity blocks using data blocks.
+	err = rs.Encode(blocks)
+	if err != nil {
+		return nil, err
+	}
+
+	// Return encoded blocks.
+	return blocks, nil
+}
+
+// appendFile - append data buffer at path.
+func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hashWriters []hash.Hash, writeQuorum int) (err error) {
+	var wg = &sync.WaitGroup{}
+	var wErrs = make([]error, len(disks))
+	// Write encoded data to quorum disks in parallel.
+	for index, disk := range disks {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		// Write encoded data in routine.
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			wErr := disk.AppendFile(volume, path, enBlocks[index])
+			if wErr != nil {
+				wErrs[index] = wErr
+				return
+			}
+
+			// Calculate hash for each blocks.
+			hashWriters[index].Write(enBlocks[index])
+
+			// Successfully wrote.
+			wErrs[index] = nil
+		}(index, disk)
+	}
+
+	// Wait for all the appends to finish.
+	wg.Wait()
+
+	// Do we have write quorum?.
+	if !isDiskQuorum(wErrs, writeQuorum) {
+		return errXLWriteQuorum
+	}
+	return nil
+}

cmd/erasure-createfile_test.go

@@ -0,0 +1,203 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/rand"
+	"testing"
+
+	"github.com/klauspost/reedsolomon"
+)
+
+// Simulates a faulty disk for AppendFile()
+type AppendDiskDown struct {
+	*posix
+}
+
+func (a AppendDiskDown) AppendFile(volume string, path string, buf []byte) error {
+	return errFaultyDisk
+}
+
+// Test erasureCreateFile()
+func TestErasureCreateFile(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 1MB with random data.
+	data := make([]byte, 1*1024*1024)
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Test when all disks are up.
+	size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	// 2 disks down.
+	disks[4] = AppendDiskDown{disks[4].(*posix)}
+	disks[5] = AppendDiskDown{disks[5].(*posix)}
+
+	// Test when two disks are down.
+	size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	// 4 more disks down. 6 disks down in total.
+	disks[6] = AppendDiskDown{disks[6].(*posix)}
+	disks[7] = AppendDiskDown{disks[7].(*posix)}
+	disks[8] = AppendDiskDown{disks[8].(*posix)}
+	disks[9] = AppendDiskDown{disks[9].(*posix)}
+
+	size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	// 1 more disk down. 7 disk down in total. Should return quorum error.
+	disks[10] = AppendDiskDown{disks[10].(*posix)}
+	_, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != errXLWriteQuorum {
+		t.Errorf("erasureCreateFile returned expected errXLWriteQuorum error, got %s", err)
+	}
+}
+
+// TestErasureEncode checks for encoding for different data sets.
+func TestErasureEncode(t *testing.T) {
+	// Collection of cases for encode cases.
+	testEncodeCases := []struct {
+		inputData         []byte
+		inputDataBlocks   int
+		inputParityBlocks int
+
+		shouldPass  bool
+		expectedErr error
+	}{
+		// TestCase - 1.
+		// Regular data encoded.
+		{
+			[]byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."),
+			8,
+			8,
+			true,
+			nil,
+		},
+		// TestCase - 2.
+		// Empty data errors out.
+		{
+			[]byte(""),
+			8,
+			8,
+			false,
+			reedsolomon.ErrShortData,
+		},
+		// TestCase - 3.
+		// Single byte encoded.
+		{
+			[]byte("1"),
+			4,
+			4,
+			true,
+			nil,
+		},
+		// TestCase - 4.
+		// test case with negative data block.
+		{
+			[]byte("1"),
+			-1,
+			8,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 5.
+		// test case with negative parity block.
+		{
+			[]byte("1"),
+			8,
+			-1,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 6.
+		// test case with zero data block.
+		{
+			[]byte("1"),
+			0,
+			8,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 7.
+		// test case with zero parity block.
+		{
+			[]byte("1"),
+			8,
+			0,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 8.
+		// test case with data + parity blocks > 255.
+		// expected to fail with Error Max Shard number.
+		{
+			[]byte("1"),
+			128,
+			128,
+			false,
+			reedsolomon.ErrMaxShardNum,
+		},
+	}
+
+	// Test encode cases.
+	for i, testCase := range testEncodeCases {
+		_, actualErr := encodeData(testCase.inputData, testCase.inputDataBlocks, testCase.inputParityBlocks)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if testCase.expectedErr != actualErr {
+				t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
+			}
+		}
+	}
+}

cmd/erasure-healfile.go

@@ -0,0 +1,84 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "encoding/hex"
+
+// Heals the erasure coded file. reedsolomon.Reconstruct() is used to reconstruct the missing parts.
+func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string, size int64, blockSize int64, dataBlocks int, parityBlocks int, algo string) (checkSums []string, err error) {
+	var offset int64
+	remainingSize := size
+
+	// Hash for bitrot protection.
+	hashWriters := newHashWriters(len(outDatedDisks), bitRotAlgo)
+
+	for remainingSize > 0 {
+		curBlockSize := blockSize
+		if remainingSize < curBlockSize {
+			curBlockSize = remainingSize
+		}
+
+		// Calculate the block size that needs to be read from each disk.
+		curEncBlockSize := getChunkSize(curBlockSize, dataBlocks)
+
+		// Memory for reading data from disks and reconstructing missing data using erasure coding.
+		enBlocks := make([][]byte, len(latestDisks))
+
+		// Read data from the latest disks.
+		// FIXME: no need to read from all the disks. dataBlocks+1 is enough.
+		for index, disk := range latestDisks {
+			if disk == nil {
+				continue
+			}
+			enBlocks[index] = make([]byte, curEncBlockSize)
+			_, err := disk.ReadFile(volume, path, offset, enBlocks[index])
+			if err != nil {
+				enBlocks[index] = nil
+			}
+		}
+
+		// Reconstruct missing data.
+		err := decodeData(enBlocks, dataBlocks, parityBlocks)
+		if err != nil {
+			return nil, err
+		}
+
+		// Write to the healPath file.
+		for index, disk := range outDatedDisks {
+			if disk == nil {
+				continue
+			}
+			err := disk.AppendFile(healBucket, healPath, enBlocks[index])
+			if err != nil {
+				return nil, err
+			}
+			hashWriters[index].Write(enBlocks[index])
+		}
+		remainingSize -= curBlockSize
+		offset += curEncBlockSize
+	}
+
+	// Checksums for the bit rot.
+	checkSums = make([]string, len(outDatedDisks))
+	for index, disk := range outDatedDisks {
+		if disk == nil {
+			continue
+		}
+		checkSums[index] = hex.EncodeToString(hashWriters[index].Sum(nil))
+	}
+	return checkSums, nil
+}

cmd/erasure-healfile_test.go

@@ -0,0 +1,123 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/rand"
+	"os"
+	"path"
+	"testing"
+)
+
+// Test erasureHealFile()
+func TestErasureHealFile(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 1MB with random data.
+	data := make([]byte, 1*1024*1024)
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Create a test file.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	latest := make([]StorageAPI, len(disks))   // Slice of latest disks
+	outDated := make([]StorageAPI, len(disks)) // Slice of outdated disks
+
+	// Test case when one part needs to be healed.
+	dataPath := path.Join(setup.diskPaths[0], "testbucket", "testobject1")
+	err = os.Remove(dataPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	copy(latest, disks)
+	latest[0] = nil
+	outDated[0] = disks[0]
+	healCheckSums, err := erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*1024*1024, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	// Checksum of the healed file should match.
+	if checkSums[0] != healCheckSums[0] {
+		t.Error("Healing failed, data does not match.")
+	}
+
+	// Test case when parityBlocks number of disks need to be healed.
+	// Should succeed.
+	copy(latest, disks)
+	for index := 0; index < parityBlocks; index++ {
+		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
+		err = os.Remove(dataPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		latest[index] = nil
+		outDated[index] = disks[index]
+	}
+
+	healCheckSums, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*1024*1024, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Checksums of the healed files should match.
+	for index := 0; index < parityBlocks; index++ {
+		if checkSums[index] != healCheckSums[index] {
+			t.Error("Healing failed, data does not match.")
+		}
+	}
+	for index := dataBlocks; index < len(disks); index++ {
+		if healCheckSums[index] != "" {
+			t.Errorf("expected healCheckSums[%d] to be empty", index)
+		}
+	}
+
+	// Test case when parityBlocks+1 number of disks need to be healed.
+	// Should fail.
+	copy(latest, disks)
+	for index := 0; index < parityBlocks+1; index++ {
+		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
+		err = os.Remove(dataPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		latest[index] = nil
+		outDated[index] = disks[index]
+	}
+	healCheckSums, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*1024*1024, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err == nil {
+		t.Error("Expected erasureHealFile() to fail when the number of available disks <= parityBlocks")
+	}
+}

cmd/erasure-readfile.go

@@ -0,0 +1,350 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/hex"
+	"errors"
+	"io"
+	"sync"
+
+	"github.com/klauspost/reedsolomon"
+	"github.com/minio/minio/pkg/bpool"
+)
+
+// isSuccessDecodeBlocks - do we have all the blocks to be
+// successfully decoded?. Input encoded blocks ordered matrix.
+func isSuccessDecodeBlocks(enBlocks [][]byte, dataBlocks int) bool {
+	// Count number of data and parity blocks that were read.
+	var successDataBlocksCount = 0
+	var successParityBlocksCount = 0
+	for index := range enBlocks {
+		if enBlocks[index] == nil {
+			continue
+		}
+		// block index lesser than data blocks, update data block count.
+		if index < dataBlocks {
+			successDataBlocksCount++
+			continue
+		} // else { // update parity block count.
+		successParityBlocksCount++
+	}
+	// Returns true if we have atleast dataBlocks parity.
+	return successDataBlocksCount == dataBlocks || successDataBlocksCount+successParityBlocksCount >= dataBlocks
+}
+
+// isSuccessDataBlocks - do we have all the data blocks?
+// Input encoded blocks ordered matrix.
+func isSuccessDataBlocks(enBlocks [][]byte, dataBlocks int) bool {
+	// Count number of data blocks that were read.
+	var successDataBlocksCount = 0
+	for index := range enBlocks[:dataBlocks] {
+		if enBlocks[index] == nil {
+			continue
+		}
+		// block index lesser than data blocks, update data block count.
+		if index < dataBlocks {
+			successDataBlocksCount++
+		}
+	}
+	// Returns true if we have atleast the dataBlocks.
+	return successDataBlocksCount >= dataBlocks
+}
+
+// Return readable disks slice from which we can read parallelly.
+func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDisks []StorageAPI, nextIndex int, err error) {
+	readDisks = make([]StorageAPI, len(orderedDisks))
+	dataDisks := 0
+	parityDisks := 0
+	// Count already read data and parity chunks.
+	for i := 0; i < index; i++ {
+		if orderedDisks[i] == nil {
+			continue
+		}
+		if i < dataBlocks {
+			dataDisks++
+		} else {
+			parityDisks++
+		}
+	}
+
+	// Sanity checks - we should never have this situation.
+	if dataDisks == dataBlocks {
+		return nil, 0, errUnexpected
+	}
+	if dataDisks+parityDisks >= dataBlocks {
+		return nil, 0, errUnexpected
+	}
+
+	// Find the disks from which next set of parallel reads should happen.
+	for i := index; i < len(orderedDisks); i++ {
+		if orderedDisks[i] == nil {
+			continue
+		}
+		if i < dataBlocks {
+			dataDisks++
+		} else {
+			parityDisks++
+		}
+		readDisks[i] = orderedDisks[i]
+		if dataDisks == dataBlocks {
+			return readDisks, i + 1, nil
+		} else if dataDisks+parityDisks == dataBlocks {
+			return readDisks, i + 1, nil
+		}
+	}
+	return nil, 0, errXLReadQuorum
+}
+
+// parallelRead - reads chunks in parallel from the disks specified in []readDisks.
+func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []StorageAPI, enBlocks [][]byte, blockOffset int64, curChunkSize int64, bitRotVerify func(diskIndex int) bool, pool *bpool.BytePool) {
+	// WaitGroup to synchronise the read go-routines.
+	wg := &sync.WaitGroup{}
+
+	// Read disks in parallel.
+	for index := range readDisks {
+		if readDisks[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		// Reads chunk from readDisk[index] in routine.
+		go func(index int) {
+			defer wg.Done()
+
+			// Verify bit rot for the file on this disk.
+			if !bitRotVerify(index) {
+				// So that we don't read from this disk for the next block.
+				orderedDisks[index] = nil
+				return
+			}
+
+			buf, err := pool.Get()
+			if err != nil {
+				errorIf(err, "unable to get buffer from byte pool")
+				orderedDisks[index] = nil
+				return
+			}
+			buf = buf[:curChunkSize]
+
+			_, err = readDisks[index].ReadFile(volume, path, blockOffset, buf)
+			if err != nil {
+				orderedDisks[index] = nil
+				return
+			}
+			enBlocks[index] = buf
+		}(index)
+	}
+
+	// Waiting for first routines to finish.
+	wg.Wait()
+}
+
+// erasureReadFile - read bytes from erasure coded files and writes to given writer.
+// Erasure coded files are read block by block as per given erasureInfo and data chunks
+// are decoded into a data block. Data block is trimmed for given offset and length,
+// then written to given writer. This function also supports bit-rot detection by
+// verifying checksum of individual block's checksum.
+func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path string, offset int64, length int64, totalLength int64, blockSize int64, dataBlocks int, parityBlocks int, checkSums []string, algo string, pool *bpool.BytePool) (int64, error) {
+	// Offset and length cannot be negative.
+	if offset < 0 || length < 0 {
+		return 0, errUnexpected
+	}
+
+	// Can't request more data than what is available.
+	if offset+length > totalLength {
+		return 0, errUnexpected
+	}
+
+	// chunkSize is the amount of data that needs to be read from each disk at a time.
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+
+	// bitRotVerify verifies if the file on a particular disk doesn't have bitrot
+	// by verifying the hash of the contents of the file.
+	bitRotVerify := func() func(diskIndex int) bool {
+		verified := make([]bool, len(disks))
+		// Return closure so that we have reference to []verified and
+		// not recalculate the hash on it every time the function is
+		// called for the same disk.
+		return func(diskIndex int) bool {
+			if verified[diskIndex] {
+				// Already validated.
+				return true
+			}
+			// Is this a valid block?
+			isValid := isValidBlock(disks[diskIndex], volume, path, checkSums[diskIndex], algo)
+			verified[diskIndex] = isValid
+			return isValid
+		}
+	}()
+
+	// Total bytes written to writer
+	bytesWritten := int64(0)
+
+	startBlock := offset / blockSize
+	endBlock := (offset + length) / blockSize
+
+	// curChunkSize = chunk size for the current block in the for loop below.
+	// curBlockSize = block size for the current block in the for loop below.
+	// curChunkSize and curBlockSize can change for the last block if totalLength%blockSize != 0
+	curChunkSize := chunkSize
+	curBlockSize := blockSize
+
+	// For each block, read chunk from each disk. If we are able to read all the data disks then we don't
+	// need to read parity disks. If one of the data disk is missing we need to read DataBlocks+1 number
+	// of disks. Once read, we Reconstruct() missing data if needed and write it to the given writer.
+	for block := startBlock; block <= endBlock; block++ {
+		// Mark all buffers as unused at the start of the loop so that the buffers
+		// can be reused.
+		pool.Reset()
+
+		// Each element of enBlocks holds curChunkSize'd amount of data read from its corresponding disk.
+		enBlocks := make([][]byte, len(disks))
+
+		if ((offset + bytesWritten) / blockSize) == (totalLength / blockSize) {
+			// This is the last block for which curBlockSize and curChunkSize can change.
+			// For ex. if totalLength is 15M and blockSize is 10MB, curBlockSize for
+			// the last block should be 5MB.
+			curBlockSize = totalLength % blockSize
+			curChunkSize = getChunkSize(curBlockSize, dataBlocks)
+		}
+
+		// NOTE: That for the offset calculation we have to use chunkSize and
+		// not curChunkSize. If we use curChunkSize for offset calculation
+		// then it can result in wrong offset for the last block.
+		blockOffset := block * chunkSize
+
+		// nextIndex - index from which next set of parallel reads
+		// should happen.
+		nextIndex := 0
+
+		for {
+			// readDisks - disks from which we need to read in parallel.
+			var readDisks []StorageAPI
+			var err error
+			// get readable disks slice from which we can read parallelly.
+			readDisks, nextIndex, err = getReadDisks(disks, nextIndex, dataBlocks)
+			if err != nil {
+				return bytesWritten, err
+			}
+			// Issue a parallel read across the disks specified in readDisks.
+			parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, bitRotVerify, pool)
+			if isSuccessDecodeBlocks(enBlocks, dataBlocks) {
+				// If enough blocks are available to do rs.Reconstruct()
+				break
+			}
+			if nextIndex == len(disks) {
+				// No more disks to read from.
+				return bytesWritten, errXLReadQuorum
+			}
+			// We do not have enough enough data blocks to reconstruct the data
+			// hence continue the for-loop till we have enough data blocks.
+		}
+
+		// If we have all the data blocks no need to decode, continue to write.
+		if !isSuccessDataBlocks(enBlocks, dataBlocks) {
+			// Reconstruct the missing data blocks.
+			if err := decodeData(enBlocks, dataBlocks, parityBlocks); err != nil {
+				return bytesWritten, err
+			}
+		}
+
+		// Offset in enBlocks from where data should be read from.
+		enBlocksOffset := int64(0)
+
+		// Total data to be read from enBlocks.
+		enBlocksLength := curBlockSize
+
+		// If this is the start block then enBlocksOffset might not be 0.
+		if block == startBlock {
+			enBlocksOffset = offset % blockSize
+			enBlocksLength -= enBlocksOffset
+		}
+
+		remaining := length - bytesWritten
+		if remaining < enBlocksLength {
+			// We should not send more data than what was requested.
+			enBlocksLength = remaining
+		}
+
+		// Write data blocks.
+		n, err := writeDataBlocks(writer, enBlocks, dataBlocks, enBlocksOffset, enBlocksLength)
+		if err != nil {
+			return bytesWritten, err
+		}
+
+		// Update total bytes written.
+		bytesWritten += n
+
+		if bytesWritten == length {
+			// Done writing all the requested data.
+			break
+		}
+	}
+
+	// Success.
+	return bytesWritten, nil
+}
+
+// isValidBlock - calculates the checksum hash for the block and
+// validates if its correct returns true for valid cases, false otherwise.
+func isValidBlock(disk StorageAPI, volume, path, checkSum, checkSumAlgo string) (ok bool) {
+	// Disk is not available, not a valid block.
+	if disk == nil {
+		return false
+	}
+	// Checksum not available, not a valid block.
+	if checkSum == "" {
+		return false
+	}
+	// Read everything for a given block and calculate hash.
+	hashWriter := newHash(checkSumAlgo)
+	hashBytes, err := hashSum(disk, volume, path, hashWriter)
+	if err != nil {
+		errorIf(err, "Unable to calculate checksum %s/%s", volume, path)
+		return false
+	}
+	return hex.EncodeToString(hashBytes) == checkSum
+}
+
+// decodeData - decode encoded blocks.
+func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
+	// Initialized reedsolomon.
+	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
+	if err != nil {
+		return err
+	}
+
+	// Reconstruct encoded blocks.
+	err = rs.Reconstruct(enBlocks)
+	if err != nil {
+		return err
+	}
+
+	// Verify reconstructed blocks (parity).
+	ok, err := rs.Verify(enBlocks)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		// Blocks cannot be reconstructed, corrupted data.
+		err = errors.New("Verification failed after reconstruction, data likely corrupted.")
+		return err
+	}
+
+	// Success.
+	return nil
+}

cmd/erasure-readfile_test.go

@@ -0,0 +1,459 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/pkg/bpool"
+)
+import "reflect"
+
+// Tests getReadDisks which returns readable disks slice from which we can
+// read parallelly.
+func testGetReadDisks(t *testing.T, xl xlObjects) {
+	d := xl.storageDisks
+	testCases := []struct {
+		index     int          // index argument for getReadDisks
+		argDisks  []StorageAPI // disks argument for getReadDisks
+		retDisks  []StorageAPI // disks return value from getReadDisks
+		nextIndex int          // return value from getReadDisks
+		err       error        // error return value from getReadDisks
+	}{
+		// Test case - 1.
+		// When all disks are available, should return data disks.
+		{
+			0,
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
+			8,
+			nil,
+		},
+		// Test case - 2.
+		// If a parity disk is down, should return all data disks.
+		{
+			0,
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], nil, d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
+			8,
+			nil,
+		},
+		// Test case - 3.
+		// If a data disk is down, should return 7 data and 1 parity.
+		{
+			0,
+			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], nil, nil, nil, nil, nil, nil, nil},
+			9,
+			nil,
+		},
+		// Test case - 4.
+		// If 7 data disks are down, should return 1 data and 7 parity.
+		{
+			0,
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], nil},
+			15,
+			nil,
+		},
+		// Test case - 5.
+		// When 2 disks fail during parallelRead, next call to getReadDisks should return 3 disks
+		{
+			8,
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, d[8], d[9], nil, nil, nil, nil, nil, nil},
+			10,
+			nil,
+		},
+		// Test case - 6.
+		// If 2 disks again fail from the 3 disks returned previously, return next 2 disks
+		{
+			11,
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, d[11], nil, nil, nil, nil},
+			12,
+			nil,
+		},
+		// Test case - 7.
+		// No more disks are available for read, return error
+		{
+			13,
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], nil, nil, nil, nil, nil},
+			nil,
+			0,
+			errXLReadQuorum,
+		},
+	}
+
+	for i, test := range testCases {
+		disks, nextIndex, err := getReadDisks(test.argDisks, test.index, xl.dataBlocks)
+		if err != test.err {
+			t.Errorf("test-case %d - expected error : %s, got : %s", i+1, test.err, err)
+			continue
+		}
+		if test.nextIndex != nextIndex {
+			t.Errorf("test-case %d - expected nextIndex: %d, got : %d", i+1, test.nextIndex, nextIndex)
+			continue
+		}
+		if reflect.DeepEqual(test.retDisks, disks) == false {
+			t.Errorf("test-case %d : incorrect disks returned. expected %+v, got %+v", i+1, test.retDisks, disks)
+			continue
+		}
+	}
+}
+
+// Test getOrderedDisks which returns ordered slice of disks from their
+// actual distribution.
+func testGetOrderedDisks(t *testing.T, xl xlObjects) {
+	disks := xl.storageDisks
+	distribution := []int{16, 14, 12, 10, 8, 6, 4, 2, 1, 3, 5, 7, 9, 11, 13, 15}
+	orderedDisks := getOrderedDisks(distribution, disks)
+	// From the "distribution" above you can notice that:
+	// 1st data block is in the 9th disk (i.e distribution index 8)
+	// 2nd data block is in the 8th disk (i.e distribution index 7) and so on.
+	if orderedDisks[0] != disks[8] ||
+		orderedDisks[1] != disks[7] ||
+		orderedDisks[2] != disks[9] ||
+		orderedDisks[3] != disks[6] ||
+		orderedDisks[4] != disks[10] ||
+		orderedDisks[5] != disks[5] ||
+		orderedDisks[6] != disks[11] ||
+		orderedDisks[7] != disks[4] ||
+		orderedDisks[8] != disks[12] ||
+		orderedDisks[9] != disks[3] ||
+		orderedDisks[10] != disks[13] ||
+		orderedDisks[11] != disks[2] ||
+		orderedDisks[12] != disks[14] ||
+		orderedDisks[13] != disks[1] ||
+		orderedDisks[14] != disks[15] ||
+		orderedDisks[15] != disks[0] {
+		t.Errorf("getOrderedDisks returned incorrect order.")
+	}
+}
+
+// Test for isSuccessDataBlocks and isSuccessDecodeBlocks.
+func TestIsSuccessBlocks(t *testing.T) {
+	dataBlocks := 8
+	testCases := []struct {
+		enBlocks      [][]byte // data and parity blocks.
+		successData   bool     // expected return value of isSuccessDataBlocks()
+		successDecode bool     // expected return value of isSuccessDecodeBlocks()
+	}{
+		{
+			// When all data and partity blocks are available.
+			[][]byte{
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			true,
+			true,
+		},
+		{
+			// When one data block is not available.
+			[][]byte{
+				nil, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// When one data and all parity are available, enough for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// When all data disks are not available, enough for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// Not enough disks for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			false,
+		},
+	}
+
+	for i, test := range testCases {
+		got := isSuccessDataBlocks(test.enBlocks, dataBlocks)
+		if test.successData != got {
+			t.Errorf("test-case %d : expected %v got %v", i+1, test.successData, got)
+		}
+		got = isSuccessDecodeBlocks(test.enBlocks, dataBlocks)
+		if test.successDecode != got {
+			t.Errorf("test-case %d : expected %v got %v", i+1, test.successDecode, got)
+		}
+	}
+}
+
+// Wrapper function for testGetReadDisks, testGetOrderedDisks.
+func TestErasureReadUtils(t *testing.T) {
+	objLayer, dirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeRoots(dirs)
+	xl := objLayer.(xlObjects)
+	testGetReadDisks(t, xl)
+	testGetOrderedDisks(t, xl)
+}
+
+// Simulates a faulty disk for ReadFile()
+type ReadDiskDown struct {
+	*posix
+}
+
+func (r ReadDiskDown) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
+	return 0, errFaultyDisk
+}
+
+func TestErasureReadFileDiskFail(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 1MB with random data.
+	data := make([]byte, 1*1024*1024)
+	length := int64(len(data))
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a test file to read from.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
+	}
+
+	// create byte pool which will be used by erasureReadFile for
+	// reading from disks and erasure decoding.
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+	pool := bpool.NewBytePool(chunkSize, len(disks))
+
+	buf := &bytes.Buffer{}
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(buf.Bytes(), data) {
+		t.Error("Contents of the erasure coded file differs")
+	}
+
+	// 2 disks down. Read should succeed.
+	disks[4] = ReadDiskDown{disks[4].(*posix)}
+	disks[5] = ReadDiskDown{disks[5].(*posix)}
+
+	buf.Reset()
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(buf.Bytes(), data) {
+		t.Error("Contents of the erasure coded file differs")
+	}
+
+	// 4 more disks down. 6 disks down in total. Read should succeed.
+	disks[6] = ReadDiskDown{disks[6].(*posix)}
+	disks[8] = ReadDiskDown{disks[8].(*posix)}
+	disks[9] = ReadDiskDown{disks[9].(*posix)}
+	disks[11] = ReadDiskDown{disks[11].(*posix)}
+
+	buf.Reset()
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(buf.Bytes(), data) {
+		t.Error("Contents of the erasure coded file differs")
+	}
+
+	// 2 more disk down. 8 disks down in total. Read should fail.
+	disks[12] = ReadDiskDown{disks[12].(*posix)}
+	disks[13] = ReadDiskDown{disks[13].(*posix)}
+	buf.Reset()
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != errXLReadQuorum {
+		t.Fatal("expected errXLReadQuorum error")
+	}
+}
+
+func TestErasureReadFileOffsetLength(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(1 * 1024 * 1024)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 5MB with random data.
+	data := make([]byte, 5*1024*1024)
+	length := int64(len(data))
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a test file to read from.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
+	}
+
+	testCases := []struct {
+		offset, length int64
+	}{
+		// Full file.
+		{0, length},
+		// Read nothing.
+		{length, 0},
+		// 2nd block.
+		{blockSize, blockSize},
+		// Test cases for random offsets and lengths.
+		{blockSize - 1, 2},
+		{blockSize - 1, blockSize + 1},
+		{blockSize + 1, blockSize - 1},
+		{blockSize + 1, blockSize},
+		{blockSize + 1, blockSize + 1},
+		{blockSize*2 - 1, blockSize + 1},
+		{length - 1, 1},
+		{length - blockSize, blockSize},
+		{length - blockSize - 1, blockSize},
+		{length - blockSize - 1, blockSize + 1},
+	}
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+	pool := bpool.NewBytePool(chunkSize, len(disks))
+
+	// Compare the data read from file with "data" byte array.
+	for i, testCase := range testCases {
+		expected := data[testCase.offset:(testCase.offset + testCase.length)]
+		buf := &bytes.Buffer{}
+		_, err = erasureReadFile(buf, disks, "testbucket", "testobject", testCase.offset, testCase.length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		got := buf.Bytes()
+		if !bytes.Equal(expected, got) {
+			t.Errorf("Test %d : read data is different from what was expected", i+1)
+		}
+	}
+}
+
+// Test erasureReadFile with random offset and lengths.
+// This test is t.Skip()ed as it a long time to run, hence should be run
+// explicitly after commenting out t.Skip()
+func TestErasureReadFileRandomOffsetLength(t *testing.T) {
+	// Comment the following line to run this test.
+	t.SkipNow()
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(1 * 1024 * 1024)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 5MB with random data.
+	data := make([]byte, 5*1024*1024)
+	length := int64(len(data))
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// 10000 iterations with random offsets and lengths.
+	iterations := 10000
+
+	// Create a test file to read from.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
+	}
+
+	// To generate random offset/length.
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	// create pool buffer which will be used by erasureReadFile for
+	// reading from disks and erasure decoding.
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+	pool := bpool.NewBytePool(chunkSize, len(disks))
+
+	buf := &bytes.Buffer{}
+
+	// Verify erasureReadFile() for random offsets and lengths.
+	for i := 0; i < iterations; i++ {
+		offset := r.Int63n(length)
+		readLen := r.Int63n(length - offset)
+
+		expected := data[offset : offset+readLen]
+
+		_, err = erasureReadFile(buf, disks, "testbucket", "testobject", offset, readLen, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+		if err != nil {
+			t.Fatal(err, offset, readLen)
+		}
+		got := buf.Bytes()
+		if !bytes.Equal(expected, got) {
+			t.Fatalf("read data is different from what was expected, offset=%d length=%d", offset, readLen)
+		}
+		buf.Reset()
+	}
+}

cmd/erasure-utils.go

@@ -0,0 +1,186 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"hash"
+	"io"
+
+	"github.com/klauspost/reedsolomon"
+	"github.com/minio/blake2b-simd"
+)
+
+// newHashWriters - inititialize a slice of hashes for the disk count.
+func newHashWriters(diskCount int, algo string) []hash.Hash {
+	hashWriters := make([]hash.Hash, diskCount)
+	for index := range hashWriters {
+		hashWriters[index] = newHash(algo)
+	}
+	return hashWriters
+}
+
+// newHash - gives you a newly allocated hash depending on the input algorithm.
+func newHash(algo string) hash.Hash {
+	switch algo {
+	case "blake2b":
+		return blake2b.New512()
+	// Add new hashes here.
+	default:
+		// Default to blake2b.
+		return blake2b.New512()
+	}
+}
+
+// hashSum calculates the hash of the entire path and returns.
+func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
+	// Allocate staging buffer of 128KiB for copyBuffer.
+	buf := make([]byte, readSizeV1)
+
+	// Copy entire buffer to writer.
+	if err := copyBuffer(writer, disk, volume, path, buf); err != nil {
+		return nil, err
+	}
+
+	// Return the final hash sum.
+	return writer.Sum(nil), nil
+}
+
+// getDataBlockLen - get length of data blocks from encoded blocks.
+func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
+	size := 0
+	// Figure out the data block length.
+	for _, block := range enBlocks[:dataBlocks] {
+		size += len(block)
+	}
+	return size
+}
+
+// Writes all the data blocks from encoded blocks until requested
+// outSize length. Provides a way to skip bytes until the offset.
+func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
+	// Offset and out size cannot be negative.
+	if offset < 0 || length < 0 {
+		return 0, errUnexpected
+	}
+
+	// Do we have enough blocks?
+	if len(enBlocks) < dataBlocks {
+		return 0, reedsolomon.ErrTooFewShards
+	}
+
+	// Do we have enough data?
+	if int64(getDataBlockLen(enBlocks, dataBlocks)) < length {
+		return 0, reedsolomon.ErrShortData
+	}
+
+	// Counter to decrement total left to write.
+	write := length
+
+	// Counter to increment total written.
+	totalWritten := int64(0)
+
+	// Write all data blocks to dst.
+	for _, block := range enBlocks[:dataBlocks] {
+		// Skip blocks until we have reached our offset.
+		if offset >= int64(len(block)) {
+			// Decrement offset.
+			offset -= int64(len(block))
+			continue
+		} else {
+			// Skip until offset.
+			block = block[offset:]
+
+			// Reset the offset for next iteration to read everything
+			// from subsequent blocks.
+			offset = 0
+		}
+		// We have written all the blocks, write the last remaining block.
+		if write < int64(len(block)) {
+			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
+			if err != nil {
+				return 0, err
+			}
+			totalWritten += n
+			break
+		}
+		// Copy the block.
+		n, err := io.Copy(dst, bytes.NewReader(block))
+		if err != nil {
+			return 0, err
+		}
+
+		// Decrement output size.
+		write -= n
+
+		// Increment written.
+		totalWritten += n
+	}
+
+	// Success.
+	return totalWritten, nil
+}
+
+// chunkSize is roughly BlockSize/DataBlocks.
+// chunkSize is calculated such that chunkSize*DataBlocks accommodates BlockSize bytes.
+// So chunkSize*DataBlocks can be slightly larger than BlockSize if BlockSize is not divisible by
+// DataBlocks. The extra space will have 0-padding.
+func getChunkSize(blockSize int64, dataBlocks int) int64 {
+	return (blockSize + int64(dataBlocks) - 1) / int64(dataBlocks)
+}
+
+// copyBuffer - copies from disk, volume, path to input writer until either EOF
+// is reached at volume, path or an error occurs. A success copyBuffer returns
+// err == nil, not err == EOF. Because copyBuffer is defined to read from path
+// until EOF. It does not treat an EOF from ReadFile an error to be reported.
+// Additionally copyBuffer stages through the provided buffer; otherwise if it
+// has zero length, returns error.
+func copyBuffer(writer io.Writer, disk StorageAPI, volume string, path string, buf []byte) error {
+	// Error condition of zero length buffer.
+	if buf != nil && len(buf) == 0 {
+		return errors.New("empty buffer in readBuffer")
+	}
+
+	// Starting offset for Reading the file.
+	startOffset := int64(0)
+
+	// Read until io.EOF.
+	for {
+		n, err := disk.ReadFile(volume, path, startOffset, buf)
+		if n > 0 {
+			m, wErr := writer.Write(buf[:n])
+			if wErr != nil {
+				return wErr
+			}
+			if int64(m) != n {
+				return io.ErrShortWrite
+			}
+		}
+		if err != nil {
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				break
+			}
+			return err
+		}
+		// Progress the offset.
+		startOffset += n
+	}
+
+	// Success.
+	return nil
+}

cmd/erasure-utils_test.go

@@ -0,0 +1,160 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"testing"
+)
+
+// Test validates the number hash writers returned.
+func TestNewHashWriters(t *testing.T) {
+	diskNum := 8
+	hashWriters := newHashWriters(diskNum, bitRotAlgo)
+	if len(hashWriters) != diskNum {
+		t.Errorf("Expected %d hashWriters, but instead got %d", diskNum, len(hashWriters))
+	}
+
+}
+
+// Tests validate the output of getChunkSize.
+func TestGetChunkSize(t *testing.T) {
+	// Refer to comments on getChunkSize() for details.
+	testCases := []struct {
+		blockSize  int64
+		dataBlocks int
+		// expected result.
+		expectedChunkSize int64
+	}{
+		{
+			10,
+			10,
+			1,
+		},
+		{
+			10,
+			11,
+			1,
+		},
+		{
+			10,
+			9,
+			2,
+		},
+	}
+	// Verify getChunkSize() for the test cases.
+	for i, testCase := range testCases {
+		got := getChunkSize(testCase.blockSize, testCase.dataBlocks)
+		if testCase.expectedChunkSize != got {
+			t.Errorf("Test %d : expected=%d got=%d", i+1, testCase.expectedChunkSize, got)
+		}
+	}
+}
+
+// TestCopyBuffer - Tests validate the result and errors produced when `copyBuffer` is called with sample inputs.
+func TestCopyBuffer(t *testing.T) {
+	// create posix test setup
+	disk, diskPath, err := newPosixTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer removeAll(diskPath)
+
+	volume := "success-vol"
+	// Setup test environment.
+	if err = disk.MakeVol(volume); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+
+	// creating io.Writer for the input to copyBuffer.
+	buffers := []*bytes.Buffer{
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+	}
+
+	testFile := "testFile"
+	testContent := []byte("hello, world")
+	err = disk.AppendFile(volume, testFile, testContent)
+	if err != nil {
+		t.Fatalf("AppendFile failed: <ERROR> %s", err)
+	}
+
+	testCases := []struct {
+		writer io.Writer
+		disk   StorageAPI
+		volume string
+		path   string
+		buf    []byte
+
+		expectedResult []byte
+		// flag to indicate whether test case should pass.
+		shouldPass  bool
+		expectedErr error
+	}{
+		// Test case - 1.
+		// case with empty buffer.
+		{nil, nil, "", "", []byte{}, nil, false, errors.New("empty buffer in readBuffer")},
+		// Test case - 2.
+		// Test case with empty volume.
+		{buffers[0], disk, "", "", make([]byte, 5), nil, false, errInvalidArgument},
+		// Test case - 3.
+		// Test case with non existent volume.
+		{buffers[0], disk, "abc", "", make([]byte, 5), nil, false, errVolumeNotFound},
+		// Test case - 4.
+		// Test case with empty filename/path.
+		{buffers[0], disk, volume, "", make([]byte, 5), nil, false, errIsNotRegular},
+		// Test case - 5.
+		// Test case with non existent file name.
+		{buffers[0], disk, volume, "abcd", make([]byte, 5), nil, false, errFileNotFound},
+		// Test case - 6.
+		// Test case where the writer returns EOF.
+		{NewEOFWriter(buffers[0], 3), disk, volume, testFile, make([]byte, 5), nil, false, io.EOF},
+		// Test case - 7.
+		// Test case to produce io.ErrShortWrite, the TruncateWriter returns after writing 3 bytes.
+		{TruncateWriter(buffers[1], 3), disk, volume, testFile, make([]byte, 5), nil, false, io.ErrShortWrite},
+		// Teset case - 8.
+		// Valid case, expected to read till EOF and write the contents into the writer.
+		{buffers[2], disk, volume, testFile, make([]byte, 5), testContent, true, nil},
+	}
+	// iterate over the test cases and call copy Buffer with data.
+	for i, testCase := range testCases {
+		actualErr := copyBuffer(testCase.writer, testCase.disk, testCase.volume, testCase.path, testCase.buf)
+
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
+		}
+
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if testCase.expectedErr.Error() != actualErr.Error() {
+				t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
+			}
+		}
+		// test passed as expected, asserting the result.
+		if actualErr == nil && testCase.shouldPass {
+			if !bytes.Equal(testCase.expectedResult, buffers[2].Bytes()) {
+				t.Errorf("Test %d: copied buffer differs from the expected one.", i+1)
+			}
+		}
+	}
+}

cmd/erasure_test.go

@@ -0,0 +1,184 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+)
+
+// mustEncodeData - encodes data slice and provides encoded 2 dimensional slice.
+func mustEncodeData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData, err := encodeData(data, dataBlocks, parityBlocks)
+	if err != nil {
+		// Upon failure panic this function.
+		panic(err)
+	}
+	return encodedData
+}
+
+// Generates good encoded data with one parity block and data block missing.
+func getGoodEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	encodedData[3] = nil
+	encodedData[1] = nil
+	return encodedData
+}
+
+// Generates bad encoded data with one parity block and data block with garbage data.
+func getBadEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	encodedData[3] = []byte("garbage")
+	encodedData[1] = []byte("garbage")
+	return encodedData
+}
+
+// Generates encoded data with all data blocks missing.
+func getMissingData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	for i := 0; i < dataBlocks+1; i++ {
+		encodedData[i] = nil
+	}
+	return encodedData
+}
+
+// Generates encoded data with less number of blocks than expected data blocks.
+func getInsufficientData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	// Return half of the data.
+	return encodedData[:dataBlocks/2]
+}
+
+// Represents erasure encoding matrix dataBlocks and paritBlocks.
+type encodingMatrix struct {
+	dataBlocks   int
+	parityBlocks int
+}
+
+// List of encoding matrices the tests will run on.
+var encodingMatrices = []encodingMatrix{
+	{3, 3}, // 3 data, 3 parity blocks.
+	{4, 4}, // 4 data, 4 parity blocks.
+	{5, 5}, // 5 data, 5 parity blocks.
+	{6, 6}, // 6 data, 6 parity blocks.
+	{7, 7}, // 7 data, 7 parity blocks.
+	{8, 8}, // 8 data, 8 parity blocks.
+}
+
+// Tests erasure decoding functionality for various types of inputs.
+func TestErasureDecode(t *testing.T) {
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+
+	// List of decoding cases
+	// - validates bad encoded data.
+	// - validates good encoded data.
+	// - validates insufficient data.
+	testDecodeCases := []struct {
+		enFn       func([]byte, int, int) [][]byte
+		shouldPass bool
+	}{
+		// Generates bad encoded data.
+		{
+			enFn:       getBadEncodedData,
+			shouldPass: false,
+		},
+		// Generates good encoded data.
+		{
+			enFn:       getGoodEncodedData,
+			shouldPass: true,
+		},
+		// Generates missing data.
+		{
+			enFn:       getMissingData,
+			shouldPass: false,
+		},
+		// Generates short data.
+		{
+			enFn:       getInsufficientData,
+			shouldPass: false,
+		},
+	}
+
+	// Validates all decode tests.
+	for i, testCase := range testDecodeCases {
+		for _, encodingMatrix := range encodingMatrices {
+
+			// Encoding matrix.
+			dataBlocks := encodingMatrix.dataBlocks
+			parityBlocks := encodingMatrix.parityBlocks
+
+			// Data block size.
+			blockSize := len(data)
+
+			// Generates encoded data based on type of testCase function.
+			encodedData := testCase.enFn(data, dataBlocks, parityBlocks)
+
+			// Decodes the data.
+			err := decodeData(encodedData, dataBlocks, parityBlocks)
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
+			}
+
+			// Proceed to extract the data blocks.
+			decodedDataWriter := new(bytes.Buffer)
+			_, err = writeDataBlocks(decodedDataWriter, encodedData, dataBlocks, 0, int64(blockSize))
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
+			}
+
+			// Validate if decoded data is what we expected.
+			if bytes.Equal(decodedDataWriter.Bytes(), data) != testCase.shouldPass {
+				err := errUnexpected
+				t.Errorf("Test %d: Expected to pass by failed instead %s", i+1, err)
+			}
+		}
+	}
+}
+
+// Setup for erasureCreateFile and erasureReadFile tests.
+type erasureTestSetup struct {
+	dataBlocks   int
+	parityBlocks int
+	blockSize    int64
+	diskPaths    []string
+	disks        []StorageAPI
+}
+
+// Removes the temporary disk directories.
+func (e erasureTestSetup) Remove() {
+	for _, path := range e.diskPaths {
+		removeAll(path)
+	}
+}
+
+// Returns an initialized setup for erasure tests.
+func newErasureTestSetup(dataBlocks int, parityBlocks int, blockSize int64) (*erasureTestSetup, error) {
+	diskPaths := make([]string, dataBlocks+parityBlocks)
+	disks := make([]StorageAPI, len(diskPaths))
+	var err error
+	for i := range diskPaths {
+		disks[i], diskPaths[i], err = newPosixTestSetup()
+		if err != nil {
+			return nil, err
+		}
+		err = disks[i].MakeVol("testbucket")
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &erasureTestSetup{dataBlocks, parityBlocks, blockSize, diskPaths, disks}, nil
+}

cmd/event-notifier.go

@@ -0,0 +1,384 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"net/url"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// Global event notification queue. This is the queue that would be used to send all notifications.
+type eventNotifier struct {
+	rwMutex *sync.RWMutex
+
+	// Collection of 'bucket' and notification config.
+	notificationConfigs map[string]*notificationConfig
+	snsTargets          map[string][]chan []NotificationEvent
+	queueTargets        map[string]*logrus.Logger
+}
+
+// Represents data to be sent with notification event.
+type eventData struct {
+	Type      EventName
+	Bucket    string
+	ObjInfo   ObjectInfo
+	ReqParams map[string]string
+}
+
+// New notification event constructs a new notification event message from
+// input request metadata which completed successfully.
+func newNotificationEvent(event eventData) NotificationEvent {
+	/// Construct a new object created event.
+	region := serverConfig.GetRegion()
+	tnow := time.Now().UTC()
+	sequencer := fmt.Sprintf("%X", tnow.UnixNano())
+	// Following blocks fills in all the necessary details of s3 event message structure.
+	// http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
+	nEvent := NotificationEvent{
+		EventVersion:      "2.0",
+		EventSource:       "aws:s3",
+		AwsRegion:         region,
+		EventTime:         tnow.Format(timeFormatAMZ),
+		EventName:         event.Type.String(),
+		UserIdentity:      defaultIdentity(),
+		RequestParameters: event.ReqParams,
+		ResponseElements:  map[string]string{},
+		S3: eventMeta{
+			SchemaVersion:   "1.0",
+			ConfigurationID: "Config",
+			Bucket: bucketMeta{
+				Name:          event.Bucket,
+				OwnerIdentity: defaultIdentity(),
+				ARN:           "arn:aws:s3:::" + event.Bucket,
+			},
+		},
+	}
+
+	escapedObj := url.QueryEscape(event.ObjInfo.Name)
+	// For delete object event type, we do not need to set ETag and Size.
+	if event.Type == ObjectRemovedDelete {
+		nEvent.S3.Object = objectMeta{
+			Key:       escapedObj,
+			Sequencer: sequencer,
+		}
+		return nEvent
+	}
+	// For all other events we should set ETag and Size.
+	nEvent.S3.Object = objectMeta{
+		Key:       escapedObj,
+		ETag:      event.ObjInfo.MD5Sum,
+		Size:      event.ObjInfo.Size,
+		Sequencer: sequencer,
+	}
+	// Success.
+	return nEvent
+}
+
+// Fetch the saved queue target.
+func (en eventNotifier) GetQueueTarget(queueARN string) *logrus.Logger {
+	return en.queueTargets[queueARN]
+}
+
+func (en eventNotifier) GetSNSTarget(snsARN string) []chan []NotificationEvent {
+	en.rwMutex.RLock()
+	defer en.rwMutex.RUnlock()
+	return en.snsTargets[snsARN]
+}
+
+// Set a new sns target for an input sns ARN.
+func (en *eventNotifier) SetSNSTarget(snsARN string, listenerCh chan []NotificationEvent) error {
+	en.rwMutex.Lock()
+	defer en.rwMutex.Unlock()
+	if listenerCh == nil {
+		return errors.New("invalid argument")
+	}
+	en.snsTargets[snsARN] = append(en.snsTargets[snsARN], listenerCh)
+	return nil
+}
+
+// Remove sns target for an input sns ARN.
+func (en *eventNotifier) RemoveSNSTarget(snsARN string, listenerCh chan []NotificationEvent) {
+	en.rwMutex.Lock()
+	defer en.rwMutex.Unlock()
+	snsTarget, ok := en.snsTargets[snsARN]
+	if ok {
+		for i, savedListenerCh := range snsTarget {
+			if listenerCh == savedListenerCh {
+				snsTarget = append(snsTarget[:i], snsTarget[i+1:]...)
+				if len(snsTarget) == 0 {
+					delete(en.snsTargets, snsARN)
+					break
+				}
+				en.snsTargets[snsARN] = snsTarget
+			}
+		}
+	}
+}
+
+// Returns true if bucket notification is set for the bucket, false otherwise.
+func (en *eventNotifier) IsBucketNotificationSet(bucket string) bool {
+	if en == nil {
+		return false
+	}
+	en.rwMutex.RLock()
+	defer en.rwMutex.RUnlock()
+	_, ok := en.notificationConfigs[bucket]
+	return ok
+}
+
+// Fetch bucket notification config for an input bucket.
+func (en eventNotifier) GetBucketNotificationConfig(bucket string) *notificationConfig {
+	en.rwMutex.RLock()
+	defer en.rwMutex.RUnlock()
+	return en.notificationConfigs[bucket]
+}
+
+// Set a new notification config for a bucket, this operation will overwrite any previous
+// notification configs for the bucket.
+func (en *eventNotifier) SetBucketNotificationConfig(bucket string, notificationCfg *notificationConfig) error {
+	en.rwMutex.Lock()
+	defer en.rwMutex.Unlock()
+	if notificationCfg == nil {
+		return errors.New("invalid argument")
+	}
+	en.notificationConfigs[bucket] = notificationCfg
+	return nil
+}
+
+// eventNotify notifies an event to relevant targets based on their
+// bucket notification configs.
+func eventNotify(event eventData) {
+	// Notifies a new event.
+	// List of events reported through this function are
+	//  - s3:ObjectCreated:Put
+	//  - s3:ObjectCreated:Post
+	//  - s3:ObjectCreated:Copy
+	//  - s3:ObjectCreated:CompleteMultipartUpload
+	//  - s3:ObjectRemoved:Delete
+
+	nConfig := eventN.GetBucketNotificationConfig(event.Bucket)
+	// No bucket notifications enabled, drop the event notification.
+	if len(nConfig.QueueConfigs) == 0 && len(nConfig.TopicConfigs) == 0 && len(nConfig.LambdaConfigs) == 0 {
+		return
+	}
+
+	// Event type.
+	eventType := event.Type.String()
+
+	// Object name.
+	objectName := event.ObjInfo.Name
+
+	// Save the notification event to be sent.
+	notificationEvent := []NotificationEvent{newNotificationEvent(event)}
+
+	// Validate if the event and object match the queue configs.
+	for _, qConfig := range nConfig.QueueConfigs {
+		eventMatch := eventMatch(eventType, qConfig.Events)
+		ruleMatch := filterRuleMatch(objectName, qConfig.Filter.Key.FilterRules)
+		if eventMatch && ruleMatch {
+			targetLog := eventN.GetQueueTarget(qConfig.QueueARN)
+			if targetLog != nil {
+				targetLog.WithFields(logrus.Fields{
+					"Records": notificationEvent,
+				}).Info()
+			}
+		}
+	}
+	// Validate if the event and object match the sns configs.
+	for _, topicConfig := range nConfig.TopicConfigs {
+		ruleMatch := filterRuleMatch(objectName, topicConfig.Filter.Key.FilterRules)
+		eventMatch := eventMatch(eventType, topicConfig.Events)
+		if eventMatch && ruleMatch {
+			targetListeners := eventN.GetSNSTarget(topicConfig.TopicARN)
+			for _, listener := range targetListeners {
+				listener <- notificationEvent
+			}
+		}
+	}
+}
+
+// loads notifcation config if any for a given bucket, returns back structured notification config.
+func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationConfig, error) {
+	// Construct the notification config path.
+	notificationConfigPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
+	objInfo, err := objAPI.GetObjectInfo(minioMetaBucket, notificationConfigPath)
+	if err != nil {
+		// 'notification.xml' not found return 'errNoSuchNotifications'.
+		// This is default when no bucket notifications are found on the bucket.
+		switch err.(type) {
+		case ObjectNotFound:
+			return nil, errNoSuchNotifications
+		}
+		// Returns error for other errors.
+		return nil, err
+	}
+	var buffer bytes.Buffer
+	err = objAPI.GetObject(minioMetaBucket, notificationConfigPath, 0, objInfo.Size, &buffer)
+	if err != nil {
+		// 'notification.xml' not found return 'errNoSuchNotifications'.
+		// This is default when no bucket notifications are found on the bucket.
+		switch err.(type) {
+		case ObjectNotFound:
+			return nil, errNoSuchNotifications
+		}
+		// Returns error for other errors.
+		return nil, err
+	}
+
+	// Unmarshal notification bytes.
+	notificationConfigBytes := buffer.Bytes()
+	notificationCfg := &notificationConfig{}
+	if err = xml.Unmarshal(notificationConfigBytes, &notificationCfg); err != nil {
+		return nil, err
+	} // Successfully marshalled notification configuration.
+
+	// Return success.
+	return notificationCfg, nil
+}
+
+// loads all bucket notifications if present.
+func loadAllBucketNotifications(objAPI ObjectLayer) (map[string]*notificationConfig, error) {
+	// List buckets to proceed loading all notification configuration.
+	buckets, err := objAPI.ListBuckets()
+	if err != nil {
+		return nil, err
+	}
+
+	configs := make(map[string]*notificationConfig)
+
+	// Loads all bucket notifications.
+	for _, bucket := range buckets {
+		var nCfg *notificationConfig
+		nCfg, err = loadNotificationConfig(bucket.Name, objAPI)
+		if err != nil {
+			if err == errNoSuchNotifications {
+				continue
+			}
+			return nil, err
+		}
+		configs[bucket.Name] = nCfg
+	}
+
+	// Success.
+	return configs, nil
+}
+
+// Loads all queue targets, initializes each queueARNs depending on their config.
+// Each instance of queueARN registers its own logrus to communicate with the
+// queue service. QueueARN once initialized is not initialized again for the
+// same queueARN, instead previous connection is used.
+func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
+	queueTargets := make(map[string]*logrus.Logger)
+	// Load all amqp targets, initialize their respective loggers.
+	for accountID, amqpN := range serverConfig.GetAMQP() {
+		if !amqpN.Enable {
+			continue
+		}
+		// Construct the queue ARN for AMQP.
+		queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueTypeAMQP
+		// Queue target if already initialized we move to the next ARN.
+		_, ok := queueTargets[queueARN]
+		if ok {
+			continue
+		}
+		// Using accountID we can now initialize a new AMQP logrus instance.
+		amqpLog, err := newAMQPNotify(accountID)
+		if err != nil {
+			return nil, err
+		}
+		queueTargets[queueARN] = amqpLog
+	}
+	// Load redis targets, initialize their respective loggers.
+	for accountID, redisN := range serverConfig.GetRedis() {
+		if !redisN.Enable {
+			continue
+		}
+		// Construct the queue ARN for Redis.
+		queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueTypeRedis
+		// Queue target if already initialized we move to the next ARN.
+		_, ok := queueTargets[queueARN]
+		if ok {
+			continue
+		}
+		// Using accountID we can now initialize a new Redis logrus instance.
+		redisLog, err := newRedisNotify(accountID)
+		if err != nil {
+			return nil, err
+		}
+		queueTargets[queueARN] = redisLog
+	}
+	// Load elastic targets, initialize their respective loggers.
+	for accountID, elasticN := range serverConfig.GetElasticSearch() {
+		if !elasticN.Enable {
+			continue
+		}
+		// Construct the queue ARN for Elastic.
+		queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueTypeElastic
+		_, ok := queueTargets[queueARN]
+		if ok {
+			continue
+		}
+		// Using accountID we can now initialize a new ElasticSearch logrus instance.
+		elasticLog, err := newElasticNotify(accountID)
+		if err != nil {
+			return nil, err
+		}
+		queueTargets[queueARN] = elasticLog
+	}
+	// Successfully initialized queue targets.
+	return queueTargets, nil
+}
+
+// Global instance of event notification queue.
+var eventN *eventNotifier
+
+// Initialize event notifier.
+func initEventNotifier(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+
+	// Read all saved bucket notifications.
+	configs, err := loadAllBucketNotifications(objAPI)
+	if err != nil {
+		return err
+	}
+
+	// Initializes all queue targets.
+	queueTargets, err := loadAllQueueTargets()
+	if err != nil {
+		return err
+	}
+
+	// Inititalize event notifier queue.
+	eventN = &eventNotifier{
+		rwMutex:             &sync.RWMutex{},
+		notificationConfigs: configs,
+		queueTargets:        queueTargets,
+		snsTargets:          make(map[string][]chan []NotificationEvent),
+	}
+
+	return nil
+}

cmd/event-notifier_test.go

@@ -0,0 +1,67 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "testing"
+
+// Tests various forms of inititalization of event notifier.
+func TestInitEventNotifier(t *testing.T) {
+	fs, disk, err := getSingleNodeObjectLayer()
+	if err != nil {
+		t.Fatal("Unable to initialize FS backend.", err)
+	}
+	xl, disks, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal("Unable to initialize XL backend.", err)
+	}
+
+	disks = append(disks, disk)
+	for _, d := range disks {
+		defer removeAll(d)
+	}
+
+	// Collection of test cases for inititalizing event notifier.
+	testCases := []struct {
+		objAPI  ObjectLayer
+		configs map[string]*notificationConfig
+		err     error
+	}{
+		// Test 1 - invalid arguments.
+		{
+			objAPI: nil,
+			err:    errInvalidArgument,
+		},
+		// Test 2 - valid FS object layer but no bucket notifications.
+		{
+			objAPI: fs,
+			err:    nil,
+		},
+		// Test 3 - valid XL object layer but no bucket notifications.
+		{
+			objAPI: xl,
+			err:    nil,
+		},
+	}
+
+	// Validate if event notifier is properly initialized.
+	for i, testCase := range testCases {
+		err = initEventNotifier(testCase.objAPI)
+		if err != testCase.err {
+			t.Errorf("Test %d: Expected %s, but got: %s", i+1, testCase.err, err)
+		}
+	}
+}

cmd/faulty-disk_test.go

@@ -0,0 +1,115 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// Simulates disk returning errFaultyDisk on all methods of StorageAPI
+// interface after successCount number of successes.
+type faultyDisk struct {
+	disk         *posix
+	successCount int
+}
+
+// instantiates a faulty
+func newFaultyDisk(disk *posix, n int) *faultyDisk {
+	return &faultyDisk{disk: disk, successCount: n}
+}
+
+func (f *faultyDisk) MakeVol(volume string) (err error) {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.MakeVol(volume)
+	}
+	return errFaultyDisk
+}
+func (f *faultyDisk) ListVols() (vols []VolInfo, err error) {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.ListVols()
+	}
+	return nil, errFaultyDisk
+}
+
+func (f *faultyDisk) StatVol(volume string) (volInfo VolInfo, err error) {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.StatVol(volume)
+	}
+	return VolInfo{}, errFaultyDisk
+}
+func (f *faultyDisk) DeleteVol(volume string) (err error) {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.DeleteVol(volume)
+	}
+	return errFaultyDisk
+}
+
+func (f *faultyDisk) ListDir(volume, path string) (entries []string, err error) {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.ListDir(volume, path)
+	}
+	return []string{}, errFaultyDisk
+}
+
+func (f *faultyDisk) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.ReadFile(volume, path, offset, buf)
+	}
+	return 0, errFaultyDisk
+}
+
+func (f *faultyDisk) AppendFile(volume, path string, buf []byte) error {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.AppendFile(volume, path, buf)
+	}
+	return errFaultyDisk
+}
+
+func (f *faultyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath)
+	}
+	return errFaultyDisk
+}
+
+func (f *faultyDisk) StatFile(volume string, path string) (file FileInfo, err error) {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.StatFile(volume, path)
+	}
+	return FileInfo{}, errFaultyDisk
+}
+
+func (f *faultyDisk) DeleteFile(volume string, path string) (err error) {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.DeleteFile(volume, path)
+	}
+	return errFaultyDisk
+}
+
+func (f *faultyDisk) ReadAll(volume string, path string) (buf []byte, err error) {
+	if f.successCount > 0 {
+		f.successCount--
+		return f.disk.ReadAll(volume, path)
+	}
+	return nil, errFaultyDisk
+}

cmd/format-config-v1.go

@@ -0,0 +1,875 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+)
+
+// fsFormat - structure holding 'fs' format.
+type fsFormat struct {
+	Version string `json:"version"`
+}
+
+// xlFormat - structure holding 'xl' format.
+type xlFormat struct {
+	Version string `json:"version"` // Version of 'xl' format.
+	Disk    string `json:"disk"`    // Disk field carries assigned disk uuid.
+	// JBOD field carries the input disk order generated the first
+	// time when fresh disks were supplied.
+	JBOD []string `json:"jbod"`
+}
+
+// formatConfigV1 - structure holds format config version '1'.
+type formatConfigV1 struct {
+	Version string `json:"version"` // Version of the format config.
+	// Format indicates the backend format type, supports two values 'xl' and 'fs'.
+	Format string    `json:"format"`
+	FS     *fsFormat `json:"fs,omitempty"` // FS field holds fs format.
+	XL     *xlFormat `json:"xl,omitempty"` // XL field holds xl format.
+}
+
+/*
+
+All disks online
+-----------------
+- All Unformatted - format all and return success.
+- Some Unformatted - format all and return success.
+- Any JBOD inconsistent - return failure // Requires deep inspection, phase2.
+- Some are corrupt (missing format.json) - return failure  // Requires deep inspection, phase2.
+- Any unrecognized disks - return failure
+
+Some disks are offline and we have quorum.
+-----------------
+- Some unformatted - no heal, return success.
+- Any JBOD inconsistent - return failure // Requires deep inspection, phase2.
+- Some are corrupt (missing format.json) - return failure  // Requires deep inspection, phase2.
+- Any unrecognized disks - return failure
+
+No read quorum
+-----------------
+failure for all cases.
+
+// Pseudo code for managing `format.json`.
+
+// Generic checks.
+if (no quorum) return error
+if (any disk is corrupt) return error // phase2
+if (jbod inconsistent) return error // phase2
+if (disks not recognized) // Always error.
+
+// Specific checks.
+if (all disks online)
+  if (all disks return format.json)
+     if (jbod consistent)
+        if (all disks recognized)
+          return
+  else
+     if (all disks return format.json not found)
+        (initialize format)
+        return
+     else (some disks return format.json not found)
+        (heal format)
+        return
+     fi
+   fi
+else // No healing at this point forward, some disks are offline or dead.
+   if (some disks return format.json not found)
+      if (with force)
+         // Offline disks are marked as dead.
+         (heal format) // Offline disks should be marked as dead.
+         return success
+      else (without force)
+         // --force is necessary to heal few drives, because some drives
+         // are offline. Offline disks will be marked as dead.
+         return error
+      fi
+fi
+*/
+
+// error returned when some disks are found to be unformatted.
+var errSomeDiskUnformatted = errors.New("some disks are found to be unformatted")
+
+// error returned when some disks are offline.
+var errSomeDiskOffline = errors.New("some disks are offline")
+
+// errDiskOrderMismatch - returned when disk UUID is not in consistent JBOD order.
+var errDiskOrderMismatch = errors.New("disk order mismatch")
+
+// Returns error slice into understandable errors.
+func reduceFormatErrs(errs []error, diskCount int) (err error) {
+	var errUnformattedDiskCount = 0
+	var errDiskNotFoundCount = 0
+	var errCorruptedFormatCount = 0
+	for _, dErr := range errs {
+		if dErr == errUnformattedDisk {
+			errUnformattedDiskCount++
+		} else if dErr == errDiskNotFound {
+			errDiskNotFoundCount++
+		} else if dErr == errCorruptedFormat {
+			errCorruptedFormatCount++
+		}
+	}
+	if errCorruptedFormatCount > 0 {
+		return errCorruptedFormat
+	}
+	// Unformatted disks found, we need to figure out if any disks are offline.
+	if errUnformattedDiskCount > 0 {
+		// Returns errUnformattedDisk if all disks report unFormattedDisk.
+		if errUnformattedDiskCount < diskCount {
+			if errDiskNotFoundCount > 0 {
+				// Only some disks are fresh but some disks are offline as well.
+				return errSomeDiskOffline
+			}
+			// Some disks are fresh disks an unformatted, not disks are offline.
+			return errSomeDiskUnformatted
+		}
+		// All disks returned unformatted, all disks must be fresh.
+		return errUnformattedDisk
+	}
+	// No unformatted disks found no need to handle disk not found case, return success here.
+	return nil
+}
+
+// loadAllFormats - load all format config from all input disks in parallel.
+func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
+	// Initialize sync waitgroup.
+	var wg = &sync.WaitGroup{}
+
+	// Initialize list of errors.
+	var sErrs = make([]error, len(bootstrapDisks))
+
+	// Initialize format configs.
+	var formatConfigs = make([]*formatConfigV1, len(bootstrapDisks))
+
+	// Make a volume entry on all underlying storage disks.
+	for index, disk := range bootstrapDisks {
+		if disk == nil {
+			sErrs[index] = errDiskNotFound
+			continue
+		}
+		wg.Add(1)
+		// Make a volume inside a go-routine.
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			formatConfig, lErr := loadFormat(disk)
+			if lErr != nil {
+				sErrs[index] = lErr
+				return
+			}
+			formatConfigs[index] = formatConfig
+		}(index, disk)
+	}
+
+	// Wait for all make vol to finish.
+	wg.Wait()
+
+	for _, err := range sErrs {
+		if err != nil {
+			// Return all formats and errors.
+			return formatConfigs, sErrs
+		}
+	}
+	// Return all formats and nil
+	return formatConfigs, nil
+}
+
+// genericFormatCheck - validates and returns error.
+// if (no quorum) return error
+// if (any disk is corrupt) return error // phase2
+// if (jbod inconsistent) return error // phase2
+// if (disks not recognized) // Always error.
+func genericFormatCheck(formatConfigs []*formatConfigV1, sErrs []error) (err error) {
+	// Calculate the errors.
+	var (
+		errCorruptFormatCount = 0
+		errCount              = 0
+	)
+
+	// Through all errors calculate the actual errors.
+	for _, lErr := range sErrs {
+		if lErr == nil {
+			continue
+		}
+		// These errors are good conditions, means disk is online.
+		if lErr == errUnformattedDisk || lErr == errVolumeNotFound {
+			continue
+		}
+		if lErr == errCorruptedFormat {
+			errCorruptFormatCount++
+		} else {
+			errCount++
+		}
+	}
+
+	// Calculate read quorum.
+	readQuorum := len(formatConfigs)/2 + 1
+
+	// Validate the err count under tolerant limit.
+	if errCount > len(formatConfigs)-readQuorum {
+		return errXLReadQuorum
+	}
+
+	// Check if number of corrupted format under quorum
+	if errCorruptFormatCount > len(formatConfigs)-readQuorum {
+		return errCorruptedFormat
+	}
+
+	// Validates if format and JBOD are consistent across all disks.
+	if err = checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	// Success..
+	return nil
+}
+
+// isSavedUUIDInOrder - validates if disk uuid is present and valid in all
+// available format config JBOD. This function also validates if the disk UUID
+// is always available on all JBOD under the same order.
+func isSavedUUIDInOrder(uuid string, formatConfigs []*formatConfigV1) bool {
+	var orderIndexes []int
+	// Validate each for format.json for relevant uuid.
+	for _, formatConfig := range formatConfigs {
+		if formatConfig == nil {
+			continue
+		}
+		// Validate if UUID is present in JBOD.
+		uuidIndex := findDiskIndex(uuid, formatConfig.XL.JBOD)
+		if uuidIndex == -1 {
+			// UUID not found.
+			errorIf(errDiskNotFound, "Disk %s not found in JBOD list", uuid)
+			return false
+		}
+		// Save the position of UUID present in JBOD.
+		orderIndexes = append(orderIndexes, uuidIndex+1)
+	}
+	// Once uuid is found, verify if the uuid
+	// present in same order across all format configs.
+	prevOrderIndex := orderIndexes[0]
+	for _, orderIndex := range orderIndexes {
+		if prevOrderIndex != orderIndex {
+			errorIf(errDiskOrderMismatch, "Disk %s is in wrong order wanted %d, saw %d ", uuid, prevOrderIndex, orderIndex)
+			return false
+		}
+	}
+	// Returns success, when we have verified if uuid
+	// is consistent and valid across all format configs.
+	return true
+}
+
+// checkDisksConsistency - checks if all disks are consistent with all JBOD entries on all disks.
+func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
+	var disks = make([]string, len(formatConfigs))
+	// Collect currently available disk uuids.
+	for index, formatConfig := range formatConfigs {
+		if formatConfig == nil {
+			disks[index] = ""
+			continue
+		}
+		disks[index] = formatConfig.XL.Disk
+	}
+	// Validate collected uuids and verify JBOD.
+	for _, uuid := range disks {
+		if uuid == "" {
+			continue
+		}
+		// Is uuid present on all JBOD ?.
+		if !isSavedUUIDInOrder(uuid, formatConfigs) {
+			return fmt.Errorf("%s disk not found in JBOD", uuid)
+		}
+	}
+	return nil
+}
+
+// checkJBODConsistency - validate xl jbod order if they are consistent.
+func checkJBODConsistency(formatConfigs []*formatConfigV1) error {
+	var sentinelJBOD []string
+	// Extract first valid JBOD.
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		sentinelJBOD = format.XL.JBOD
+		break
+	}
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		currentJBOD := format.XL.JBOD
+		if !reflect.DeepEqual(sentinelJBOD, currentJBOD) {
+			return errors.New("Inconsistent JBOD found.")
+		}
+	}
+	return nil
+}
+
+// findDiskIndex returns position of disk in JBOD.
+func findDiskIndex(disk string, jbod []string) int {
+	for index, uuid := range jbod {
+		if uuid == disk {
+			return index
+		}
+	}
+	return -1
+}
+
+// reorderDisks - reorder disks in JBOD order.
+func reorderDisks(bootstrapDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
+	var savedJBOD []string
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		savedJBOD = format.XL.JBOD
+		break
+	}
+	// Pick the first JBOD list to verify the order and construct new set of disk slice.
+	var newDisks = make([]StorageAPI, len(bootstrapDisks))
+	for fIndex, format := range formatConfigs {
+		if format == nil {
+			continue
+		}
+		jIndex := findDiskIndex(format.XL.Disk, savedJBOD)
+		if jIndex == -1 {
+			return nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found")
+		}
+		newDisks[jIndex] = bootstrapDisks[fIndex]
+	}
+	return newDisks, nil
+}
+
+// loadFormat - loads format.json from disk.
+func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
+	buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
+	if err != nil {
+		// 'file not found' and 'volume not found' as
+		// same. 'volume not found' usually means its a fresh disk.
+		if err == errFileNotFound || err == errVolumeNotFound {
+			var vols []VolInfo
+			vols, err = disk.ListVols()
+			if err != nil {
+				return nil, err
+			}
+			if len(vols) > 1 {
+				// 'format.json' not found, but we found user data.
+				return nil, errCorruptedFormat
+			}
+			// No other data found, its a fresh disk.
+			return nil, errUnformattedDisk
+		}
+		return nil, err
+	}
+
+	// Try to decode format json into formatConfigV1 struct.
+	format = &formatConfigV1{}
+	if err = json.Unmarshal(buf, format); err != nil {
+		return nil, err
+	}
+
+	// Success.
+	return format, nil
+}
+
+// isFormatNotFound - returns true if all `format.json` are not
+// found on all disks.
+func isFormatNotFound(formats []*formatConfigV1) bool {
+	for _, format := range formats {
+		// One of the `format.json` is found.
+		if format != nil {
+			return false
+		}
+	}
+	// All format.json missing, success.
+	return true
+}
+
+// isFormatFound - returns true if all input formats are found on
+// all disks.
+func isFormatFound(formats []*formatConfigV1) bool {
+	for _, format := range formats {
+		// One of `format.json` is not found.
+		if format == nil {
+			return false
+		}
+	}
+	// All format.json present, success.
+	return true
+}
+
+// Heals any missing format.json on the drives. Returns error only for unexpected errors
+// as regular errors can be ignored since there might be enough quorum to be operational.
+// Heals only fresh disks.
+func healFormatXLFreshDisks(storageDisks []StorageAPI) error {
+	formatConfigs := make([]*formatConfigV1, len(storageDisks))
+	var referenceConfig *formatConfigV1
+	// Loads `format.json` from all disks.
+	for index, disk := range storageDisks {
+		// Disk not found or ignored is a valid case.
+		if disk == nil {
+			// Return nil, one of the disk is offline.
+			return nil
+		}
+		formatXL, err := loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk {
+				// format.json is missing, should be healed.
+				continue
+			} else if err == errDiskNotFound { // Is a valid case we
+				// can proceed without healing.
+				return nil
+			}
+			// Return error for unsupported errors.
+			return err
+		} // Success.
+		formatConfigs[index] = formatXL
+	}
+
+	// All `format.json` has been read successfully, previously completed.
+	if isFormatFound(formatConfigs) {
+		// Return success.
+		return nil
+	}
+
+	// All disks are fresh, format.json will be written by initFormatXL()
+	if isFormatNotFound(formatConfigs) {
+		return initFormatXL(storageDisks)
+	}
+
+	// Validate format configs for consistency in JBOD and disks.
+	if err := checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	if referenceConfig == nil {
+		// This config will be used to update the drives missing format.json.
+		for _, formatConfig := range formatConfigs {
+			if formatConfig == nil {
+				continue
+			}
+			referenceConfig = formatConfig
+			break
+		}
+	}
+
+	// Collect new JBOD.
+	newJBOD := referenceConfig.XL.JBOD
+
+	// Reorder the disks based on the JBOD order.
+	orderedDisks, err := reorderDisks(storageDisks, formatConfigs)
+	if err != nil {
+		return err
+	}
+
+	// From ordered disks fill the UUID position.
+	for index, disk := range orderedDisks {
+		if disk == nil {
+			newJBOD[index] = getUUID()
+		}
+	}
+
+	// Collect new format configs.
+	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
+
+	// Collect new format configs that need to be written.
+	for index := range orderedDisks {
+		// New configs are generated since we are going
+		// to re-populate across all disks.
+		config := &formatConfigV1{
+			Version: referenceConfig.Version,
+			Format:  referenceConfig.Format,
+			XL: &xlFormat{
+				Version: referenceConfig.XL.Version,
+				Disk:    newJBOD[index],
+				JBOD:    newJBOD,
+			},
+		}
+		newFormatConfigs[index] = config
+	}
+
+	// Fill in the missing disk back from format configs.
+	// We need to make sure we have kept the previous order
+	// and allowed fresh disks to be arranged anywhere.
+	// Following block facilitates to put fresh disks.
+	for index, format := range formatConfigs {
+		// Format is missing so we go through ordered disks.
+		if format == nil {
+			// At this point when disk is missing the fresh disk
+			// in the stack get it back from storageDisks.
+			for oIndex, disk := range orderedDisks {
+				if disk == nil {
+					orderedDisks[oIndex] = storageDisks[index]
+					break
+				}
+			}
+		}
+	}
+
+	// Save new `format.json` across all disks, in JBOD order.
+	return saveFormatXL(orderedDisks, newFormatConfigs)
+}
+
+// Disks from storageDiks are put in assignedDisks if found in orderedDisks and in unAssignedDisks otherwise
+func splitDisksByUse(storageDisks, orderedDisks []StorageAPI) (assignedDisks []StorageAPI, unAssignedDisks []StorageAPI) {
+	// Populate unAssignDisks
+	for i := range storageDisks {
+		found := false
+		for j := range orderedDisks {
+			if storageDisks[i] == orderedDisks[j] {
+				found = true
+				assignedDisks = append(assignedDisks, storageDisks[i])
+				break
+			}
+		}
+		if !found {
+			unAssignedDisks = append(unAssignedDisks, storageDisks[i])
+		}
+	}
+	return assignedDisks, unAssignedDisks
+}
+
+// Inspect the content of all disks to guess the right order according to the format files.
+// The right order is represented in orderedDisks
+func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
+	for index, format := range formatConfigs {
+		if format != nil {
+			continue
+		}
+		vols, err := storageDisks[index].ListVols()
+		if err != nil {
+			return nil, err
+		}
+		if len(vols) == 0 {
+			continue
+		}
+		volName := ""
+		// Avoid picking minioMetaBucket because ListVols() returns a non ordered list
+		for i := range vols {
+			if vols[i].Name != minioMetaBucket {
+				volName = vols[i].Name
+				break
+			}
+		}
+		if volName == "" {
+			continue
+		}
+		objects, err := storageDisks[index].ListDir(volName, "")
+		if err != nil {
+			return nil, err
+		}
+		if len(objects) == 0 {
+			continue
+		}
+		xlData, err := readXLMeta(storageDisks[index], volName, objects[0])
+		if err != nil {
+			if err == errFileNotFound {
+				continue
+			}
+			return nil, err
+		}
+		diskIndex := -1
+		for i, d := range xlData.Erasure.Distribution {
+			if d == xlData.Erasure.Index {
+				diskIndex = i
+			}
+		}
+		// Check for found results
+		if diskIndex == -1 || orderedDisks[diskIndex] != nil {
+			// Some inconsistent data are found, exit immediately.
+			return nil, errCorruptedFormat
+		}
+		orderedDisks[diskIndex] = storageDisks[index]
+	}
+	return orderedDisks, nil
+}
+
+// Heals corrupted format json in all disks
+func healFormatXLCorruptedDisks(storageDisks []StorageAPI) error {
+	formatConfigs := make([]*formatConfigV1, len(storageDisks))
+	var referenceConfig *formatConfigV1
+
+	// Loads `format.json` from all disks.
+	for index, disk := range storageDisks {
+		// Disk not found or ignored is a valid case.
+		if disk == nil {
+			// Return nil, one of the disk is offline.
+			return nil
+		}
+		formatXL, err := loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk || err == errCorruptedFormat {
+				// format.json is missing or corrupted, should be healed.
+				continue
+			} else if err == errDiskNotFound { // Is a valid case we
+				// can proceed without healing.
+				return nil
+			}
+			// Return error for unsupported errors.
+			return err
+		} // Success.
+		formatConfigs[index] = formatXL
+	}
+
+	// All `format.json` has been read successfully, previously completed.
+	if isFormatFound(formatConfigs) {
+		// Return success.
+		return nil
+	}
+
+	// All disks are fresh, format.json will be written by initFormatXL()
+	if isFormatNotFound(formatConfigs) {
+		return initFormatXL(storageDisks)
+	}
+
+	// Validate format configs for consistency in JBOD and disks.
+	if err := checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	if referenceConfig == nil {
+		// This config will be used to update the drives missing format.json.
+		for _, formatConfig := range formatConfigs {
+			if formatConfig == nil {
+				continue
+			}
+			referenceConfig = formatConfig
+			break
+		}
+	}
+
+	// Collect new JBOD.
+	newJBOD := referenceConfig.XL.JBOD
+
+	// Reorder the disks based on the JBOD order.
+	orderedDisks, err := reorderDisks(storageDisks, formatConfigs)
+	if err != nil {
+		return err
+	}
+
+	// From ordered disks fill the UUID position.
+	for index, disk := range orderedDisks {
+		if disk == nil {
+			newJBOD[index] = getUUID()
+		}
+	}
+
+	// For disks with corrupted formats, inspect the disks contents to guess the disks order
+	orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks, formatConfigs)
+	if err != nil {
+		return err
+	}
+
+	// At this stage, all disks with corrupted formats but with objects inside found their way.
+	// Now take care of unformatted disks, which are the `unAssignedDisks`
+	_, unAssignedDisks := splitDisksByUse(storageDisks, orderedDisks)
+
+	// Assign unassigned disks to nil elements in orderedDisks
+	for i, disk := range orderedDisks {
+		if disk == nil && len(unAssignedDisks) > 0 {
+			orderedDisks[i] = unAssignedDisks[0]
+			unAssignedDisks = unAssignedDisks[1:]
+		}
+	}
+
+	// Collect new format configs.
+	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
+
+	// Collect new format configs that need to be written.
+	for index := range orderedDisks {
+		// New configs are generated since we are going
+		// to re-populate across all disks.
+		config := &formatConfigV1{
+			Version: referenceConfig.Version,
+			Format:  referenceConfig.Format,
+			XL: &xlFormat{
+				Version: referenceConfig.XL.Version,
+				Disk:    newJBOD[index],
+				JBOD:    newJBOD,
+			},
+		}
+		newFormatConfigs[index] = config
+	}
+
+	// Save new `format.json` across all disks, in JBOD order.
+	return saveFormatXL(orderedDisks, newFormatConfigs)
+}
+
+// loadFormatXL - loads XL `format.json` and returns back properly
+// ordered storage slice based on `format.json`.
+func loadFormatXL(bootstrapDisks []StorageAPI) (disks []StorageAPI, err error) {
+	var unformattedDisksFoundCnt = 0
+	var diskNotFoundCount = 0
+	formatConfigs := make([]*formatConfigV1, len(bootstrapDisks))
+
+	// Try to load `format.json` bootstrap disks.
+	for index, disk := range bootstrapDisks {
+		if disk == nil {
+			diskNotFoundCount++
+			continue
+		}
+		var formatXL *formatConfigV1
+		formatXL, err = loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk {
+				unformattedDisksFoundCnt++
+				continue
+			} else if err == errDiskNotFound {
+				diskNotFoundCount++
+				continue
+			}
+			return nil, err
+		}
+		// Save valid formats.
+		formatConfigs[index] = formatXL
+	}
+
+	// If all disks indicate that 'format.json' is not available return 'errUnformattedDisk'.
+	if unformattedDisksFoundCnt > len(bootstrapDisks)-(len(bootstrapDisks)/2+1) {
+		return nil, errUnformattedDisk
+	} else if diskNotFoundCount == len(bootstrapDisks) {
+		return nil, errDiskNotFound
+	} else if diskNotFoundCount > len(bootstrapDisks)-(len(bootstrapDisks)/2+1) {
+		return nil, errXLReadQuorum
+	}
+
+	// Validate the format configs read are correct.
+	if err = checkFormatXL(formatConfigs); err != nil {
+		return nil, err
+	}
+	// Erasure code requires disks to be presented in the same order each time.
+	return reorderDisks(bootstrapDisks, formatConfigs)
+}
+
+// checkFormatXL - verifies if format.json format is intact.
+func checkFormatXL(formatConfigs []*formatConfigV1) error {
+	for _, formatXL := range formatConfigs {
+		if formatXL == nil {
+			continue
+		}
+		// Validate format version and format type.
+		if formatXL.Version != "1" {
+			return fmt.Errorf("Unsupported version of backend format [%s] found.", formatXL.Version)
+		}
+		if formatXL.Format != "xl" {
+			return fmt.Errorf("Unsupported backend format [%s] found.", formatXL.Format)
+		}
+		if formatXL.XL.Version != "1" {
+			return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version)
+		}
+		if len(formatConfigs) != len(formatXL.XL.JBOD) {
+			return fmt.Errorf("Number of disks %d did not match the backend format %d", len(formatConfigs), len(formatXL.XL.JBOD))
+		}
+	}
+	if err := checkJBODConsistency(formatConfigs); err != nil {
+		return err
+	}
+	return checkDisksConsistency(formatConfigs)
+}
+
+// saveFormatXL - populates `format.json` on disks in its order.
+func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error {
+	var errs = make([]error, len(storageDisks))
+	var wg = &sync.WaitGroup{}
+	// Write `format.json` to all disks.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int, disk StorageAPI, format *formatConfigV1) {
+			defer wg.Done()
+
+			// Marshal and write to disk.
+			formatBytes, err := json.Marshal(format)
+			if err != nil {
+				errs[index] = err
+				return
+			}
+
+			// Purge any existing temporary file, okay to ignore errors here.
+			disk.DeleteFile(minioMetaBucket, formatConfigFileTmp)
+
+			// Append file `format.json.tmp`.
+			if err = disk.AppendFile(minioMetaBucket, formatConfigFileTmp, formatBytes); err != nil {
+				errs[index] = err
+				return
+			}
+			// Rename file `format.json.tmp` --> `format.json`.
+			if err = disk.RenameFile(minioMetaBucket, formatConfigFileTmp, minioMetaBucket, formatConfigFile); err != nil {
+				errs[index] = err
+				return
+			}
+		}(index, disk, formats[index])
+	}
+
+	// Wait for the routines to finish.
+	wg.Wait()
+
+	// Validate if we encountered any errors, return quickly.
+	for _, err := range errs {
+		if err != nil {
+			// Failure.
+			return err
+		}
+	}
+
+	// Success.
+	return nil
+}
+
+// initFormatXL - save XL format configuration on all disks.
+func initFormatXL(storageDisks []StorageAPI) (err error) {
+	// Initialize jbods.
+	var jbod = make([]string, len(storageDisks))
+
+	// Initialize formats.
+	var formats = make([]*formatConfigV1, len(storageDisks))
+
+	// Initialize `format.json`.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		// Allocate format config.
+		formats[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    getUUID(),
+			},
+		}
+		jbod[index] = formats[index].XL.Disk
+	}
+
+	// Update the jbod entries.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		// Save jbod.
+		formats[index].XL.JBOD = jbod
+	}
+
+	// Save formats `format.json` across all disks.
+	return saveFormatXL(storageDisks, formats)
+}

cmd/format-config-v1_test.go

@@ -0,0 +1,445 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+)
+
+// generates a valid format.json for XL backend.
+func genFormatXLValid() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidVersion() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Corrupt version numbers.
+	formatConfigs[0].Version = "2"
+	formatConfigs[3].Version = "-1"
+	return formatConfigs
+}
+
+// generates a invalid format.json JBOD for XL backend.
+func genFormatXLInvalidJBOD() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	// Corrupt JBOD entries on disk 6 and disk 8.
+	formatConfigs[5].XL.JBOD = jbod
+	formatConfigs[7].XL.JBOD = jbod
+	return formatConfigs
+}
+
+// generates a invalid format.json Disk UUID for XL backend.
+func genFormatXLInvalidDisks() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Make disk 5 and disk 8 have inconsistent disk uuid's.
+	formatConfigs[4].XL.Disk = getUUID()
+	formatConfigs[7].XL.Disk = getUUID()
+	return formatConfigs
+}
+
+// generates a invalid format.json Disk UUID in wrong order for XL backend.
+func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Re order jbod for failure case.
+	var jbod1 = make([]string, 8)
+	for i, j := range jbod {
+		jbod1[i] = j
+	}
+	jbod1[1], jbod1[2] = jbod[2], jbod[1]
+	formatConfigs[2].XL.JBOD = jbod1
+	return formatConfigs
+}
+
+func TestFormatXLHealFreshDisks(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	xl := obj.(xlObjects)
+
+	err = obj.MakeBucket("bucket")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := "bucket"
+	object := "object"
+
+	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	/* // Now, remove two format files.. Load them and reorder
+	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[11].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	} */
+
+	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
+	// created when minio starts
+	for i := 3; i <= 5; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			t.Fatal(err)
+		}
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "tmp"); err != nil {
+			t.Fatal(err)
+		}
+		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/xl.json"); err != nil {
+			t.Fatal(err)
+		}
+		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/part.1"); err != nil {
+			t.Fatal(err)
+		}
+		if err = xl.storageDisks[i].DeleteVol(bucket); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	permutedStorageDisks := []StorageAPI{xl.storageDisks[1], xl.storageDisks[4],
+		xl.storageDisks[2], xl.storageDisks[8], xl.storageDisks[6], xl.storageDisks[7],
+		xl.storageDisks[0], xl.storageDisks[15], xl.storageDisks[13], xl.storageDisks[14],
+		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
+		xl.storageDisks[5], xl.storageDisks[11]}
+
+	// Start healing disks
+	err = healFormatXLFreshDisks(permutedStorageDisks)
+	if err != nil {
+		t.Fatal("healing corrupted disk failed: ", err)
+	}
+
+	// Load again XL format.json to validate it
+	_, err = loadFormatXL(permutedStorageDisks)
+	if err != nil {
+		t.Fatal("loading healed disk failed: ", err)
+	}
+
+	// Clean all
+	removeRoots(fsDirs)
+}
+
+// Simulate XL disks creation, delete some format.json and remove the content of
+// a given disk to test healing a corrupted disk
+func TestFormatXLHealCorruptedDisks(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	xl := obj.(xlObjects)
+
+	err = obj.MakeBucket("bucket")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := "bucket"
+	object := "object"
+
+	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Now, remove two format files.. Load them and reorder
+	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[11].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
+	// created when minio starts
+	if err = xl.storageDisks[10].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[10].DeleteFile(".minio.sys", "tmp"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[10].DeleteFile(bucket, object+"/xl.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[10].DeleteFile(bucket, object+"/part.1"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[10].DeleteVol(bucket); err != nil {
+		t.Fatal(err)
+	}
+
+	permutedStorageDisks := []StorageAPI{xl.storageDisks[1], xl.storageDisks[4],
+		xl.storageDisks[2], xl.storageDisks[8], xl.storageDisks[6], xl.storageDisks[7],
+		xl.storageDisks[0], xl.storageDisks[15], xl.storageDisks[13], xl.storageDisks[14],
+		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
+		xl.storageDisks[5], xl.storageDisks[11]}
+
+	// Start healing disks
+	err = healFormatXLCorruptedDisks(permutedStorageDisks)
+	if err != nil {
+		t.Fatal("healing corrupted disk failed: ", err)
+	}
+
+	// Load again XL format.json to validate it
+	_, err = loadFormatXL(permutedStorageDisks)
+	if err != nil {
+		t.Fatal("loading healed disk failed: ", err)
+	}
+
+	// Clean all
+	removeRoots(fsDirs)
+}
+
+// Test on ReorderByInspection by simulating creating disks and removing
+// some of format.json
+func TestFormatXLReorderByInspection(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	xl := obj.(xlObjects)
+
+	err = obj.MakeBucket("bucket")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := "bucket"
+	object := "object"
+
+	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Now, remove two format files.. Load them and reorder
+	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+	if err = xl.storageDisks[5].DeleteFile(".minio.sys", "format.json"); err != nil {
+		t.Fatal(err)
+	}
+
+	permutedStorageDisks := []StorageAPI{xl.storageDisks[1], xl.storageDisks[4],
+		xl.storageDisks[2], xl.storageDisks[8], xl.storageDisks[6], xl.storageDisks[7],
+		xl.storageDisks[0], xl.storageDisks[15], xl.storageDisks[13], xl.storageDisks[14],
+		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
+		xl.storageDisks[5], xl.storageDisks[11]}
+
+	permutedFormatConfigs, _ := loadAllFormats(permutedStorageDisks)
+
+	orderedDisks, err := reorderDisks(permutedStorageDisks, permutedFormatConfigs)
+	if err != nil {
+		t.Fatal("error reordering disks\n")
+	}
+
+	orderedDisks, err = reorderDisksByInspection(orderedDisks, permutedStorageDisks, permutedFormatConfigs)
+	if err != nil {
+		t.Fatal("failed to reorder disk by inspection")
+	}
+
+	// Check disks reordering
+	for i := 0; i <= 15; i++ {
+		if orderedDisks[i] == nil && i != 3 && i != 5 {
+			t.Fatal("should not be nil")
+		}
+		if orderedDisks[i] != nil && orderedDisks[i] != xl.storageDisks[i] {
+			t.Fatal("Disks were not ordered correctly.")
+		}
+	}
+
+	removeRoots(fsDirs)
+}
+
+// Wrapper for calling FormatXL tests - currently validates
+//  - valid format
+//  - unrecognized version number
+//  - invalid JBOD
+//  - invalid Disk uuid
+func TestFormatXL(t *testing.T) {
+	formatInputCases := [][]*formatConfigV1{
+		genFormatXLValid(),
+		genFormatXLInvalidVersion(),
+		genFormatXLInvalidJBOD(),
+		genFormatXLInvalidDisks(),
+		genFormatXLInvalidDisksOrder(),
+	}
+	testCases := []struct {
+		formatConfigs []*formatConfigV1
+		shouldPass    bool
+	}{
+		{
+			formatConfigs: formatInputCases[0],
+			shouldPass:    true,
+		},
+		{
+			formatConfigs: formatInputCases[1],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[2],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[3],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[4],
+			shouldPass:    false,
+		},
+	}
+
+	for i, testCase := range testCases {
+		err := checkFormatXL(testCase.formatConfigs)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass but failed with %s", i+1, err)
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail but passed instead", i+1)
+		}
+	}
+}
+
+// Tests uuid order verification function.
+func TestSavedUUIDOrder(t *testing.T) {
+	uuidTestCases := make([]struct {
+		uuid       string
+		shouldPass bool
+	}, 8)
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+		uuidTestCases[index].uuid = jbod[index]
+		uuidTestCases[index].shouldPass = true
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Re order jbod for failure case.
+	var jbod1 = make([]string, 8)
+	for i, j := range jbod {
+		jbod1[i] = j
+	}
+	jbod1[1], jbod1[2] = jbod[2], jbod[1]
+	formatConfigs[2].XL.JBOD = jbod1
+	uuidTestCases[1].shouldPass = false
+	uuidTestCases[2].shouldPass = false
+
+	for i, testCase := range uuidTestCases {
+		// Is uuid present on all JBOD ?.
+		if testCase.shouldPass != isSavedUUIDInOrder(testCase.uuid, formatConfigs) {
+			t.Errorf("Test %d: Expected to pass but failed", i+1)
+		}
+	}
+}

cmd/fs-createfile.go

@@ -0,0 +1,41 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "io"
+
+func fsCreateFile(disk StorageAPI, reader io.Reader, buf []byte, tmpBucket, tempObj string) (int64, error) {
+	bytesWritten := int64(0)
+	// Read the buffer till io.EOF and append the read data to the temporary file.
+	for {
+		n, rErr := reader.Read(buf)
+		if rErr != nil && rErr != io.EOF {
+			return 0, rErr
+		}
+		bytesWritten += int64(n)
+		if n > 0 {
+			wErr := disk.AppendFile(tmpBucket, tempObj, buf[0:n])
+			if wErr != nil {
+				return 0, wErr
+			}
+		}
+		if rErr == io.EOF {
+			break
+		}
+	}
+	return bytesWritten, nil
+}

cmd/fs-v1-errors.go

@@ -0,0 +1,22 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "errors"
+
+// errFSDiskFormat - returned when given disk format is other than FS format.
+var errFSDiskFormat = errors.New("Disk is not in FS format.")

cmd/fs-v1-metadata.go

@@ -0,0 +1,174 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"os"
+	"path"
+	"sort"
+	"strings"
+)
+
+const (
+	fsMetaJSONFile   = "fs.json"
+	fsFormatJSONFile = "format.json"
+)
+
+// A fsMetaV1 represents a metadata header mapping keys to sets of values.
+type fsMetaV1 struct {
+	Version string `json:"version"`
+	Format  string `json:"format"`
+	Minio   struct {
+		Release string `json:"release"`
+	} `json:"minio"`
+	// Metadata map for current object `fs.json`.
+	Meta  map[string]string `json:"meta,omitempty"`
+	Parts []objectPartInfo  `json:"parts,omitempty"`
+}
+
+// ObjectPartIndex - returns the index of matching object part number.
+func (m fsMetaV1) ObjectPartIndex(partNumber int) (partIndex int) {
+	for i, part := range m.Parts {
+		if partNumber == part.Number {
+			partIndex = i
+			return partIndex
+		}
+	}
+	return -1
+}
+
+// AddObjectPart - add a new object part in order.
+func (m *fsMetaV1) AddObjectPart(partNumber int, partName string, partETag string, partSize int64) {
+	partInfo := objectPartInfo{
+		Number: partNumber,
+		Name:   partName,
+		ETag:   partETag,
+		Size:   partSize,
+	}
+
+	// Update part info if it already exists.
+	for i, part := range m.Parts {
+		if partNumber == part.Number {
+			m.Parts[i] = partInfo
+			return
+		}
+	}
+
+	// Proceed to include new part info.
+	m.Parts = append(m.Parts, partInfo)
+
+	// Parts in fsMeta should be in sorted order by part number.
+	sort.Sort(byObjectPartNumber(m.Parts))
+}
+
+// readFSMetadata - returns the object metadata `fs.json` content.
+func readFSMetadata(disk StorageAPI, bucket, object string) (fsMeta fsMetaV1, err error) {
+	// Read all `fs.json`.
+	buf, err := disk.ReadAll(bucket, path.Join(object, fsMetaJSONFile))
+	if err != nil {
+		return fsMetaV1{}, err
+	}
+
+	// Decode `fs.json` into fsMeta structure.
+	if err = json.Unmarshal(buf, &fsMeta); err != nil {
+		return fsMetaV1{}, err
+	}
+
+	// Success.
+	return fsMeta, nil
+}
+
+// newFSMetaV1 - initializes new fsMetaV1.
+func newFSMetaV1() (fsMeta fsMetaV1) {
+	fsMeta = fsMetaV1{}
+	fsMeta.Version = "1.0.0"
+	fsMeta.Format = "fs"
+	fsMeta.Minio.Release = ReleaseTag
+	return fsMeta
+}
+
+// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
+func newFSFormatV1() (format formatConfigV1) {
+	return formatConfigV1{
+		Version: "1",
+		Format:  "fs",
+		FS: &fsFormat{
+			Version: "1",
+		},
+	}
+}
+
+// isFSFormat - returns whether given formatConfigV1 is FS type or not.
+func isFSFormat(format formatConfigV1) bool {
+	return format.Format == "fs"
+}
+
+// writes FS format (format.json) into minioMetaBucket.
+func writeFSFormatData(storage StorageAPI, fsFormat formatConfigV1) error {
+	metadataBytes, err := json.Marshal(fsFormat)
+	if err != nil {
+		return err
+	}
+	// fsFormatJSONFile - format.json file stored in minioMetaBucket(.minio) directory.
+	if err = storage.AppendFile(minioMetaBucket, fsFormatJSONFile, metadataBytes); err != nil {
+		return err
+	}
+	return nil
+}
+
+// writeFSMetadata - writes `fs.json` metadata, marshals fsMeta object into json
+// and saves it to disk.
+func writeFSMetadata(storage StorageAPI, bucket, path string, fsMeta fsMetaV1) error {
+	metadataBytes, err := json.Marshal(fsMeta)
+	if err != nil {
+		return err
+	}
+	if err = storage.AppendFile(bucket, path, metadataBytes); err != nil {
+		return err
+	}
+	return nil
+}
+
+var extendedHeaders = []string{
+	"X-Amz-Meta-",
+	"X-Minio-Meta-",
+	// Add new extended headers.
+}
+
+// isExtendedHeader validates if input string matches extended headers.
+func isExtendedHeader(header string) bool {
+	for _, extendedHeader := range extendedHeaders {
+		if strings.HasPrefix(header, extendedHeader) {
+			return true
+		}
+	}
+	return false
+}
+
+// Return true if extended HTTP headers are set, false otherwise.
+func hasExtendedHeader(metadata map[string]string) bool {
+	if os.Getenv("MINIO_ENABLE_FSMETA") == "1" {
+		return true
+	}
+	for k := range metadata {
+		if isExtendedHeader(k) {
+			return true
+		}
+	}
+	return false
+}

cmd/fs-v1-metadata_test.go

@@ -0,0 +1,66 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"testing"
+)
+
+// Tests scenarios which can occur for hasExtendedHeader function.
+func TestHasExtendedHeader(t *testing.T) {
+	// All test cases concerning hasExtendedHeader function.
+	testCases := []struct {
+		metadata map[string]string
+		has      bool
+	}{
+		// Verifies if X-Amz-Meta is present.
+		{
+			metadata: map[string]string{
+				"X-Amz-Meta-1": "value",
+			},
+			has: true || os.Getenv("MINIO_ENABLE_FSMETA") == "1",
+		},
+		// Verifies if X-Minio-Meta is present.
+		{
+			metadata: map[string]string{
+				"X-Minio-Meta-1": "value",
+			},
+			has: true || os.Getenv("MINIO_ENABLE_FSMETA") == "1",
+		},
+		// Verifies if extended header is not present.
+		{
+			metadata: map[string]string{
+				"md5Sum": "value",
+			},
+			has: false || os.Getenv("MINIO_ENABLE_FSMETA") == "1",
+		},
+		// Verifies if extended header is not present, but with an empty input.
+		{
+			metadata: nil,
+			has:      false || os.Getenv("MINIO_ENABLE_FSMETA") == "1",
+		},
+	}
+
+	// Validate all test cases.
+	for i, testCase := range testCases {
+		has := hasExtendedHeader(testCase.metadata)
+		if has != testCase.has {
+			t.Fatalf("Test case %d: Expected \"%#v\", but got \"%#v\"", i+1, testCase.has, has)
+		}
+	}
+}

cmd/fs-v1-multipart-common.go

@@ -0,0 +1,112 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"path"
+	"time"
+)
+
+// Returns if the prefix is a multipart upload.
+func (fs fsObjects) isMultipartUpload(bucket, prefix string) bool {
+	_, err := fs.storage.StatFile(bucket, pathJoin(prefix, uploadsJSONFile))
+	return err == nil
+}
+
+// Checks whether bucket exists.
+func (fs fsObjects) isBucketExist(bucket string) bool {
+	// Check whether bucket exists.
+	_, err := fs.storage.StatVol(bucket)
+	if err != nil {
+		if err == errVolumeNotFound {
+			return false
+		}
+		errorIf(err, "Stat failed on bucket "+bucket+".")
+		return false
+	}
+	return true
+}
+
+// isUploadIDExists - verify if a given uploadID exists and is valid.
+func (fs fsObjects) isUploadIDExists(bucket, object, uploadID string) bool {
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	_, err := fs.storage.StatFile(minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
+	if err != nil {
+		if err == errFileNotFound {
+			return false
+		}
+		errorIf(err, "Unable to access upload id"+uploadIDPath)
+		return false
+	}
+	return true
+}
+
+// writeUploadJSON - create `uploads.json` or update it with new uploadID.
+func (fs fsObjects) writeUploadJSON(bucket, object, uploadID string, initiated time.Time) (err error) {
+	uploadsPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
+	uniqueID := getUUID()
+	tmpUploadsPath := path.Join(tmpMetaPrefix, uniqueID)
+	var uploadsJSON uploadsV1
+	uploadsJSON, err = readUploadsJSON(bucket, object, fs.storage)
+	if err != nil {
+		// For any other errors.
+		if err != errFileNotFound {
+			return err
+		}
+		// Set uploads format to `fs`.
+		uploadsJSON = newUploadsV1("fs")
+	}
+	// Add a new upload id.
+	uploadsJSON.AddUploadID(uploadID, initiated)
+
+	// Update `uploads.json` on all disks.
+	uploadsJSONBytes, wErr := json.Marshal(&uploadsJSON)
+	if wErr != nil {
+		return wErr
+	}
+	// Write `uploads.json` to disk.
+	if wErr = fs.storage.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsJSONBytes); wErr != nil {
+		return wErr
+	}
+	wErr = fs.storage.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath)
+	if wErr != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tmpUploadsPath); dErr != nil {
+			return dErr
+		}
+		return wErr
+	}
+	return nil
+}
+
+// updateUploadsJSON - update `uploads.json` with new uploadsJSON for all disks.
+func (fs fsObjects) updateUploadsJSON(bucket, object string, uploadsJSON uploadsV1) (err error) {
+	uploadsPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
+	uniqueID := getUUID()
+	tmpUploadsPath := path.Join(tmpMetaPrefix, uniqueID)
+	uploadsBytes, wErr := json.Marshal(uploadsJSON)
+	if wErr != nil {
+		return wErr
+	}
+	if wErr = fs.storage.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsBytes); wErr != nil {
+		return wErr
+	}
+	if wErr = fs.storage.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath); wErr != nil {
+		return wErr
+	}
+	return nil
+}

cmd/fs-v1-multipart.go

@@ -0,0 +1,714 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/skyrings/skyring-common/tools/uuid"
+)
+
+// listMultipartUploads - lists all multipart uploads.
+func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+	result := ListMultipartsInfo{}
+	recursive := true
+	if delimiter == slashSeparator {
+		recursive = false
+	}
+
+	result.IsTruncated = true
+	result.MaxUploads = maxUploads
+	result.KeyMarker = keyMarker
+	result.Prefix = prefix
+	result.Delimiter = delimiter
+
+	// Not using path.Join() as it strips off the trailing '/'.
+	multipartPrefixPath := pathJoin(mpartMetaPrefix, bucket, prefix)
+	if prefix == "" {
+		// Should have a trailing "/" if prefix is ""
+		// For ex. multipartPrefixPath should be "multipart/bucket/" if prefix is ""
+		multipartPrefixPath += slashSeparator
+	}
+	multipartMarkerPath := ""
+	if keyMarker != "" {
+		multipartMarkerPath = pathJoin(mpartMetaPrefix, bucket, keyMarker)
+	}
+	var uploads []uploadMetadata
+	var err error
+	var eof bool
+	if uploadIDMarker != "" {
+		nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker))
+		uploads, _, err = listMultipartUploadIDs(bucket, keyMarker, uploadIDMarker, maxUploads, fs.storage)
+		nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, keyMarker))
+		if err != nil {
+			return ListMultipartsInfo{}, err
+		}
+		maxUploads = maxUploads - len(uploads)
+	}
+	var walkResultCh chan treeWalkResult
+	var endWalkCh chan struct{}
+	heal := false // true only for xl.ListObjectsHeal()
+	if maxUploads > 0 {
+		walkResultCh, endWalkCh = fs.listPool.Release(listParams{minioMetaBucket, recursive, multipartMarkerPath, multipartPrefixPath, heal})
+		if walkResultCh == nil {
+			endWalkCh = make(chan struct{})
+			isLeaf := fs.isMultipartUpload
+			listDir := listDirFactory(isLeaf, fs.storage)
+			walkResultCh = startTreeWalk(minioMetaBucket, multipartPrefixPath, multipartMarkerPath, recursive, listDir, isLeaf, endWalkCh)
+		}
+		for maxUploads > 0 {
+			walkResult, ok := <-walkResultCh
+			if !ok {
+				// Closed channel.
+				eof = true
+				break
+			}
+			// For any walk error return right away.
+			if walkResult.err != nil {
+				// File not found or Disk not found is a valid case.
+				if isErrIgnored(walkResult.err, walkResultIgnoredErrs) {
+					eof = true
+					break
+				}
+				return ListMultipartsInfo{}, err
+			}
+			entry := strings.TrimPrefix(walkResult.entry, retainSlash(pathJoin(mpartMetaPrefix, bucket)))
+			if strings.HasSuffix(walkResult.entry, slashSeparator) {
+				uploads = append(uploads, uploadMetadata{
+					Object: entry,
+				})
+				maxUploads--
+				if maxUploads == 0 {
+					if walkResult.end {
+						eof = true
+						break
+					}
+				}
+				continue
+			}
+			var tmpUploads []uploadMetadata
+			var end bool
+			uploadIDMarker = ""
+			nsMutex.RLock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry))
+			tmpUploads, end, err = listMultipartUploadIDs(bucket, entry, uploadIDMarker, maxUploads, fs.storage)
+			nsMutex.RUnlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, entry))
+			if err != nil {
+				return ListMultipartsInfo{}, err
+			}
+			uploads = append(uploads, tmpUploads...)
+			maxUploads -= len(tmpUploads)
+			if walkResult.end && end {
+				eof = true
+				break
+			}
+		}
+	}
+	// Loop through all the received uploads fill in the multiparts result.
+	for _, upload := range uploads {
+		var objectName string
+		var uploadID string
+		if strings.HasSuffix(upload.Object, slashSeparator) {
+			// All directory entries are common prefixes.
+			uploadID = "" // Upload ids are empty for CommonPrefixes.
+			objectName = upload.Object
+			result.CommonPrefixes = append(result.CommonPrefixes, objectName)
+		} else {
+			uploadID = upload.UploadID
+			objectName = upload.Object
+			result.Uploads = append(result.Uploads, upload)
+		}
+		result.NextKeyMarker = objectName
+		result.NextUploadIDMarker = uploadID
+	}
+
+	if !eof {
+		// Save the go-routine state in the pool so that it can continue from where it left off on
+		// the next request.
+		fs.listPool.Set(listParams{bucket, recursive, result.NextKeyMarker, prefix, heal}, walkResultCh, endWalkCh)
+	}
+
+	result.IsTruncated = !eof
+	if !result.IsTruncated {
+		result.NextKeyMarker = ""
+		result.NextUploadIDMarker = ""
+	}
+	return result, nil
+}
+
+// ListMultipartUploads - lists all the pending multipart uploads on a
+// bucket. Additionally takes 'prefix, keyMarker, uploadIDmarker and a
+// delimiter' which allows us to list uploads match a particular
+// prefix or lexically starting from 'keyMarker' or delimiting the
+// output to get a directory like listing.
+//
+// Implements S3 compatible ListMultipartUploads API. The resulting
+// ListMultipartsInfo structure is unmarshalled directly into XML and
+// replied back to the client.
+func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
+	// Validate input arguments.
+	if !IsValidBucketName(bucket) {
+		return ListMultipartsInfo{}, BucketNameInvalid{Bucket: bucket}
+	}
+	if !fs.isBucketExist(bucket) {
+		return ListMultipartsInfo{}, BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectPrefix(prefix) {
+		return ListMultipartsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix}
+	}
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != slashSeparator {
+		return ListMultipartsInfo{}, UnsupportedDelimiter{
+			Delimiter: delimiter,
+		}
+	}
+	// Verify if marker has prefix.
+	if keyMarker != "" && !strings.HasPrefix(keyMarker, prefix) {
+		return ListMultipartsInfo{}, InvalidMarkerPrefixCombination{
+			Marker: keyMarker,
+			Prefix: prefix,
+		}
+	}
+	if uploadIDMarker != "" {
+		if strings.HasSuffix(keyMarker, slashSeparator) {
+			return ListMultipartsInfo{}, InvalidUploadIDKeyCombination{
+				UploadIDMarker: uploadIDMarker,
+				KeyMarker:      keyMarker,
+			}
+		}
+		id, err := uuid.Parse(uploadIDMarker)
+		if err != nil {
+			return ListMultipartsInfo{}, err
+		}
+		if id.IsZero() {
+			return ListMultipartsInfo{}, MalformedUploadID{
+				UploadID: uploadIDMarker,
+			}
+		}
+	}
+	return fs.listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
+}
+
+// newMultipartUpload - wrapper for initializing a new multipart
+// request, returns back a unique upload id.
+//
+// Internally this function creates 'uploads.json' associated for the
+// incoming object at '.minio/multipart/bucket/object/uploads.json' on
+// all the disks. `uploads.json` carries metadata regarding on going
+// multipart operation on the object.
+func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[string]string) (uploadID string, err error) {
+	// Initialize `fs.json` values.
+	fsMeta := newFSMetaV1()
+
+	// Save additional metadata only if extended headers such as "X-Amz-Meta-" are set.
+	if hasExtendedHeader(meta) {
+		fsMeta.Meta = meta
+	}
+
+	// This lock needs to be held for any changes to the directory contents of ".minio/multipart/object/"
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object))
+
+	uploadID = getUUID()
+	initiated := time.Now().UTC()
+	// Create 'uploads.json'
+	if err = fs.writeUploadJSON(bucket, object, uploadID, initiated); err != nil {
+		return "", err
+	}
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	tempFSMetadataPath := path.Join(tmpMetaPrefix, getUUID()+"-"+fsMetaJSONFile)
+	if err = writeFSMetadata(fs.storage, minioMetaBucket, tempFSMetadataPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, tempFSMetadataPath)
+	}
+	err = fs.storage.RenameFile(minioMetaBucket, tempFSMetadataPath, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
+	if err != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempFSMetadataPath); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tempFSMetadataPath)
+		}
+		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+	// Return success.
+	return uploadID, nil
+}
+
+// NewMultipartUpload - initialize a new multipart upload, returns a
+// unique id. The unique id returned here is of UUID form, for each
+// subsequent request each UUID is unique.
+//
+// Implements S3 compatible initiate multipart API.
+func (fs fsObjects) NewMultipartUpload(bucket, object string, meta map[string]string) (string, error) {
+	// Verify if bucket name is valid.
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify whether the bucket exists.
+	if !fs.isBucketExist(bucket) {
+		return "", BucketNotFound{Bucket: bucket}
+	}
+	// Verify if object name is valid.
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+	return fs.newMultipartUpload(bucket, object, meta)
+}
+
+// PutObjectPart - reads incoming data until EOF for the part file on
+// an ongoing multipart transaction. Internally incoming data is
+// written to '.minio/tmp' location and safely renamed to
+// '.minio/multipart' for reach parts.
+func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify whether the bucket exists.
+	if !fs.isBucketExist(bucket) {
+		return "", BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+
+	nsMutex.RLock(minioMetaBucket, uploadIDPath)
+	// Just check if the uploadID exists to avoid copy if it doesn't.
+	uploadIDExists := fs.isUploadIDExists(bucket, object, uploadID)
+	nsMutex.RUnlock(minioMetaBucket, uploadIDPath)
+	if !uploadIDExists {
+		return "", InvalidUploadID{UploadID: uploadID}
+	}
+
+	// Hold write lock on the part so that there is no parallel upload on the part.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(partID)))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(partID)))
+
+	partSuffix := fmt.Sprintf("object%d", partID)
+	tmpPartPath := path.Join(tmpMetaPrefix, uploadID+"-"+partSuffix)
+
+	// Initialize md5 writer.
+	md5Writer := md5.New()
+
+	// Limit the reader to its provided size if specified.
+	var limitDataReader io.Reader
+	if size > 0 {
+		// This is done so that we can avoid erroneous clients sending more data than the set content size.
+		limitDataReader = io.LimitReader(data, size)
+	} else {
+		// else we read till EOF.
+		limitDataReader = data
+	}
+
+	teeReader := io.TeeReader(limitDataReader, md5Writer)
+	bufSize := int64(readSizeV1)
+	if size > 0 && bufSize > size {
+		bufSize = size
+	}
+	buf := make([]byte, int(bufSize))
+	bytesWritten, cErr := fsCreateFile(fs.storage, teeReader, buf, minioMetaBucket, tmpPartPath)
+	if cErr != nil {
+		fs.storage.DeleteFile(minioMetaBucket, tmpPartPath)
+		return "", toObjectErr(cErr, minioMetaBucket, tmpPartPath)
+	}
+	// Should return IncompleteBody{} error when reader has fewer
+	// bytes than specified in request header.
+	if bytesWritten < size {
+		fs.storage.DeleteFile(minioMetaBucket, tmpPartPath)
+		return "", IncompleteBody{}
+	}
+
+	// Validate if payload is valid.
+	if isSignVerify(data) {
+		if err := data.(*signVerifyReader).Verify(); err != nil {
+			// Incoming payload wrong, delete the temporary object.
+			fs.storage.DeleteFile(minioMetaBucket, tmpPartPath)
+			// Error return.
+			return "", toObjectErr(err, bucket, object)
+		}
+	}
+
+	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
+	if md5Hex != "" {
+		if newMD5Hex != md5Hex {
+			// MD5 mismatch, delete the temporary object.
+			fs.storage.DeleteFile(minioMetaBucket, tmpPartPath)
+			// Returns md5 mismatch.
+			return "", BadDigest{md5Hex, newMD5Hex}
+		}
+	}
+
+	// Hold write lock as we are updating fs.json
+	nsMutex.Lock(minioMetaBucket, uploadIDPath)
+	defer nsMutex.Unlock(minioMetaBucket, uploadIDPath)
+
+	// Just check if the uploadID exists to avoid copy if it doesn't.
+	if !fs.isUploadIDExists(bucket, object, uploadID) {
+		return "", InvalidUploadID{UploadID: uploadID}
+	}
+
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
+	if err != nil {
+		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+	fsMeta.AddObjectPart(partID, partSuffix, newMD5Hex, size)
+
+	partPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, partSuffix)
+	err = fs.storage.RenameFile(minioMetaBucket, tmpPartPath, minioMetaBucket, partPath)
+	if err != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tmpPartPath); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tmpPartPath)
+		}
+		return "", toObjectErr(err, minioMetaBucket, partPath)
+	}
+	uploadIDPath = path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	tempFSMetadataPath := path.Join(tmpMetaPrefix, getUUID()+"-"+fsMetaJSONFile)
+	if err = writeFSMetadata(fs.storage, minioMetaBucket, tempFSMetadataPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, tempFSMetadataPath)
+	}
+	err = fs.storage.RenameFile(minioMetaBucket, tempFSMetadataPath, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
+	if err != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempFSMetadataPath); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tempFSMetadataPath)
+		}
+		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+	return newMD5Hex, nil
+}
+
+// listObjectParts - wrapper scanning through
+// '.minio/multipart/bucket/object/UPLOADID'. Lists all the parts
+// saved inside '.minio/multipart/bucket/object/UPLOADID'.
+func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
+	result := ListPartsInfo{}
+
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
+	if err != nil {
+		return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+	// Only parts with higher part numbers will be listed.
+	partIdx := fsMeta.ObjectPartIndex(partNumberMarker)
+	parts := fsMeta.Parts
+	if partIdx != -1 {
+		parts = fsMeta.Parts[partIdx+1:]
+	}
+	count := maxParts
+	for _, part := range parts {
+		var fi FileInfo
+		partNamePath := path.Join(mpartMetaPrefix, bucket, object, uploadID, part.Name)
+		fi, err = fs.storage.StatFile(minioMetaBucket, partNamePath)
+		if err != nil {
+			return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, partNamePath)
+		}
+		result.Parts = append(result.Parts, partInfo{
+			PartNumber:   part.Number,
+			ETag:         part.ETag,
+			LastModified: fi.ModTime,
+			Size:         fi.Size,
+		})
+		count--
+		if count == 0 {
+			break
+		}
+	}
+	// If listed entries are more than maxParts, we set IsTruncated as true.
+	if len(parts) > len(result.Parts) {
+		result.IsTruncated = true
+		// Make sure to fill next part number marker if IsTruncated is
+		// true for subsequent listing.
+		nextPartNumberMarker := result.Parts[len(result.Parts)-1].PartNumber
+		result.NextPartNumberMarker = nextPartNumberMarker
+	}
+	result.Bucket = bucket
+	result.Object = object
+	result.UploadID = uploadID
+	result.MaxParts = maxParts
+	return result, nil
+}
+
+// ListObjectParts - lists all previously uploaded parts for a given
+// object and uploadID.  Takes additional input of part-number-marker
+// to indicate where the listing should begin from.
+//
+// Implements S3 compatible ListObjectParts API. The resulting
+// ListPartsInfo structure is unmarshalled directly into XML and
+// replied back to the client.
+func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return ListPartsInfo{}, BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify whether the bucket exists.
+	if !fs.isBucketExist(bucket) {
+		return ListPartsInfo{}, BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return ListPartsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+	// Hold lock so that there is no competing abort-multipart-upload or complete-multipart-upload.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+
+	if !fs.isUploadIDExists(bucket, object, uploadID) {
+		return ListPartsInfo{}, InvalidUploadID{UploadID: uploadID}
+	}
+	return fs.listObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
+}
+
+// CompleteMultipartUpload - completes an ongoing multipart
+// transaction after receiving all the parts indicated by the client.
+// Returns an md5sum calculated by concatenating all the individual
+// md5sums of all the parts.
+//
+// Implements S3 compatible Complete multipart API.
+func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (string, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify whether the bucket exists.
+	if !fs.isBucketExist(bucket) {
+		return "", BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{
+			Bucket: bucket,
+			Object: object,
+		}
+	}
+
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	// Hold lock so that
+	// 1) no one aborts this multipart upload
+	// 2) no one does a parallel complete-multipart-upload on this
+	// multipart upload
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+
+	if !fs.isUploadIDExists(bucket, object, uploadID) {
+		return "", InvalidUploadID{UploadID: uploadID}
+	}
+
+	// Read saved fs metadata for ongoing multipart.
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
+	if err != nil {
+		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	}
+
+	// Calculate s3 compatible md5sum for complete multipart.
+	s3MD5, err := completeMultipartMD5(parts...)
+	if err != nil {
+		return "", err
+	}
+
+	tempObj := path.Join(tmpMetaPrefix, uploadID+"-"+"part.1")
+
+	// Allocate staging buffer.
+	var buf = make([]byte, readSizeV1)
+
+	// Loop through all parts, validate them and then commit to disk.
+	for i, part := range parts {
+		partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
+		if partIdx == -1 {
+			return "", InvalidPart{}
+		}
+		if fsMeta.Parts[partIdx].ETag != part.ETag {
+			return "", BadDigest{}
+		}
+		// All parts except the last part has to be atleast 5MB.
+		if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
+			return "", PartTooSmall{
+				PartNumber: part.PartNumber,
+				PartSize:   fsMeta.Parts[partIdx].Size,
+				PartETag:   part.ETag,
+			}
+		}
+		// Construct part suffix.
+		partSuffix := fmt.Sprintf("object%d", part.PartNumber)
+		multipartPartFile := path.Join(mpartMetaPrefix, bucket, object, uploadID, partSuffix)
+		offset := int64(0)
+		totalLeft := fsMeta.Parts[partIdx].Size
+		for totalLeft > 0 {
+			curLeft := int64(readSizeV1)
+			if totalLeft < readSizeV1 {
+				curLeft = totalLeft
+			}
+			var n int64
+			n, err = fs.storage.ReadFile(minioMetaBucket, multipartPartFile, offset, buf[:curLeft])
+			if n > 0 {
+				if err = fs.storage.AppendFile(minioMetaBucket, tempObj, buf[:n]); err != nil {
+					return "", toObjectErr(err, minioMetaBucket, tempObj)
+				}
+			}
+			if err != nil {
+				if err == io.EOF || err == io.ErrUnexpectedEOF {
+					break
+				}
+				if err == errFileNotFound {
+					return "", InvalidPart{}
+				}
+				return "", toObjectErr(err, minioMetaBucket, multipartPartFile)
+			}
+			offset += n
+			totalLeft -= n
+		}
+	}
+
+	// Rename the file back to original location, if not delete the temporary object.
+	err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
+	if err != nil {
+		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempObj); dErr != nil {
+			return "", toObjectErr(dErr, minioMetaBucket, tempObj)
+		}
+		return "", toObjectErr(err, bucket, object)
+	}
+
+	// No need to save part info, since we have concatenated all parts.
+	fsMeta.Parts = nil
+
+	// Save additional metadata only if extended headers such as "X-Amz-Meta-" are set.
+	if hasExtendedHeader(fsMeta.Meta) {
+		if len(fsMeta.Meta) == 0 {
+			fsMeta.Meta = make(map[string]string)
+		}
+		fsMeta.Meta["md5Sum"] = s3MD5
+
+		// Write the metadata to a temp file and rename it to the actual location.
+		tmpMetaPath := path.Join(tmpMetaPrefix, getUUID())
+		if err = writeFSMetadata(fs.storage, minioMetaBucket, tmpMetaPath, fsMeta); err != nil {
+			return "", toObjectErr(err, bucket, object)
+		}
+		fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
+		if err = fs.storage.RenameFile(minioMetaBucket, tmpMetaPath, minioMetaBucket, fsMetaPath); err != nil {
+			return "", toObjectErr(err, bucket, object)
+		}
+	}
+
+	// Cleanup all the parts if everything else has been safely committed.
+	if err = cleanupUploadedParts(bucket, object, uploadID, fs.storage); err != nil {
+		return "", toObjectErr(err, bucket, object)
+	}
+
+	// Hold the lock so that two parallel complete-multipart-uploads do not
+	// leave a stale uploads.json behind.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object))
+
+	// Validate if there are other incomplete upload-id's present for
+	// the object, if yes do not attempt to delete 'uploads.json'.
+	uploadsJSON, err := readUploadsJSON(bucket, object, fs.storage)
+	if err != nil {
+		return "", toObjectErr(err, minioMetaBucket, object)
+	}
+	// If we have successfully read `uploads.json`, then we proceed to
+	// purge or update `uploads.json`.
+	uploadIDIdx := uploadsJSON.Index(uploadID)
+	if uploadIDIdx != -1 {
+		uploadsJSON.Uploads = append(uploadsJSON.Uploads[:uploadIDIdx], uploadsJSON.Uploads[uploadIDIdx+1:]...)
+	}
+	if len(uploadsJSON.Uploads) > 0 {
+		if err = fs.updateUploadsJSON(bucket, object, uploadsJSON); err != nil {
+			return "", toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object))
+		}
+		// Return success.
+		return s3MD5, nil
+	}
+
+	if err = fs.storage.DeleteFile(minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object))
+	}
+
+	// Return md5sum.
+	return s3MD5, nil
+}
+
+// abortMultipartUpload - wrapper for purging an ongoing multipart
+// transaction, deletes uploadID entry from `uploads.json` and purges
+// the directory at '.minio/multipart/bucket/object/uploadID' holding
+// all the upload parts.
+func (fs fsObjects) abortMultipartUpload(bucket, object, uploadID string) error {
+	// Cleanup all uploaded parts.
+	if err := cleanupUploadedParts(bucket, object, uploadID, fs.storage); err != nil {
+		return err
+	}
+
+	// Validate if there are other incomplete upload-id's present for
+	// the object, if yes do not attempt to delete 'uploads.json'.
+	uploadsJSON, err := readUploadsJSON(bucket, object, fs.storage)
+	if err == nil {
+		uploadIDIdx := uploadsJSON.Index(uploadID)
+		if uploadIDIdx != -1 {
+			uploadsJSON.Uploads = append(uploadsJSON.Uploads[:uploadIDIdx], uploadsJSON.Uploads[uploadIDIdx+1:]...)
+		}
+		// There are pending uploads for the same object, preserve
+		// them update 'uploads.json' in-place.
+		if len(uploadsJSON.Uploads) > 0 {
+			err = fs.updateUploadsJSON(bucket, object, uploadsJSON)
+			if err != nil {
+				return toObjectErr(err, bucket, object)
+			}
+			return nil
+		}
+	} // No more pending uploads for the object, we purge the entire
+	// entry at '.minio/multipart/bucket/object'.
+	if err = fs.storage.DeleteFile(minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)); err != nil {
+		return toObjectErr(err, minioMetaBucket, path.Join(mpartMetaPrefix, bucket, object))
+	}
+	return nil
+}
+
+// AbortMultipartUpload - aborts an ongoing multipart operation
+// signified by the input uploadID. This is an atomic operation
+// doesn't require clients to initiate multiple such requests.
+//
+// All parts are purged from all disks and reference to the uploadID
+// would be removed from the system, rollback is not possible on this
+// operation.
+//
+// Implements S3 compatible Abort multipart API, slight difference is
+// that this is an atomic idempotent operation. Subsequent calls have
+// no affect and further requests to the same uploadID would not be
+// honored.
+func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	if !fs.isBucketExist(bucket) {
+		return BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+
+	// Hold lock so that there is no competing complete-multipart-upload or put-object-part.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+
+	if !fs.isUploadIDExists(bucket, object, uploadID) {
+		return InvalidUploadID{UploadID: uploadID}
+	}
+
+	err := fs.abortMultipartUpload(bucket, object, uploadID)
+	return err
+}

cmd/fs-v1.go

@@ -0,0 +1,665 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/mimedb"
+)
+
+// fsObjects - Implements fs object layer.
+type fsObjects struct {
+	storage      StorageAPI
+	physicalDisk string
+
+	// List pool management.
+	listPool *treeWalkPool
+}
+
+// creates format.json, the FS format info in minioMetaBucket.
+func initFormatFS(storageDisk StorageAPI) error {
+	return writeFSFormatData(storageDisk, newFSFormatV1())
+}
+
+// loads format.json from minioMetaBucket if it exists.
+func loadFormatFS(storageDisk StorageAPI) (format formatConfigV1, err error) {
+	// Reads entire `format.json`.
+	buf, err := storageDisk.ReadAll(minioMetaBucket, fsFormatJSONFile)
+	if err != nil {
+		return formatConfigV1{}, err
+	}
+
+	// Unmarshal format config.
+	if err = json.Unmarshal(buf, &format); err != nil {
+		return formatConfigV1{}, err
+	}
+
+	// Return structured `format.json`.
+	return format, nil
+}
+
+// newFSObjects - initialize new fs object layer.
+func newFSObjects(disk string) (ObjectLayer, error) {
+	storage, err := newStorageAPI(disk)
+	if err != nil && err != errDiskNotFound {
+		return nil, err
+	}
+
+	// Attempt to create `.minio`.
+	err = storage.MakeVol(minioMetaBucket)
+	if err != nil {
+		switch err {
+		// Ignore the errors.
+		case errVolumeExists, errDiskNotFound, errFaultyDisk:
+		default:
+			return nil, toObjectErr(err, minioMetaBucket)
+		}
+	}
+
+	// Runs house keeping code, like creating minioMetaBucket, cleaning up tmp files etc.
+	if err = fsHouseKeeping(storage); err != nil {
+		return nil, err
+	}
+
+	// loading format.json from minioMetaBucket.
+	// Note: The format.json content is ignored, reserved for future use.
+	format, err := loadFormatFS(storage)
+	if err != nil {
+		if err == errFileNotFound {
+			// format.json doesn't exist, create it inside minioMetaBucket.
+			err = initFormatFS(storage)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	} else if !isFSFormat(format) {
+		return nil, errFSDiskFormat
+	}
+
+	// Initialize fs objects.
+	fs := fsObjects{
+		storage:      storage,
+		physicalDisk: disk,
+		listPool:     newTreeWalkPool(globalLookupTimeout),
+	}
+
+	// Return successfully initialized object layer.
+	return fs, nil
+}
+
+// Should be called when process shuts down.
+func (fs fsObjects) Shutdown() error {
+	// List if there are any multipart entries.
+	_, err := fs.storage.ListDir(minioMetaBucket, mpartMetaPrefix)
+	if err != errFileNotFound {
+		// Multipart directory is not empty hence do not remove '.minio.sys' volume.
+		return nil
+	}
+	// List if there are any bucket configuration entries.
+	_, err = fs.storage.ListDir(minioMetaBucket, bucketConfigPrefix)
+	if err != errFileNotFound {
+		// Bucket config directory is not empty hence do not remove '.minio.sys' volume.
+		return nil
+	}
+	// Cleanup everything else.
+	prefix := ""
+	if err = cleanupDir(fs.storage, minioMetaBucket, prefix); err != nil {
+		errorIf(err, "Unable to cleanup minio meta bucket")
+		return err
+	}
+	if err = fs.storage.DeleteVol(minioMetaBucket); err != nil {
+		if err != errVolumeNotEmpty {
+			errorIf(err, "Unable to delete minio meta bucket %s", minioMetaBucket)
+			return err
+		}
+	}
+	// Successful.
+	return nil
+}
+
+// StorageInfo - returns underlying storage statistics.
+func (fs fsObjects) StorageInfo() StorageInfo {
+	info, err := disk.GetInfo(fs.physicalDisk)
+	fatalIf(err, "Unable to get disk info "+fs.physicalDisk)
+	return StorageInfo{
+		Total: info.Total,
+		Free:  info.Free,
+	}
+}
+
+/// Bucket operations
+
+// MakeBucket - make a bucket.
+func (fs fsObjects) MakeBucket(bucket string) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	if err := fs.storage.MakeVol(bucket); err != nil {
+		return toObjectErr(err, bucket)
+	}
+	return nil
+}
+
+// GetBucketInfo - get bucket info.
+func (fs fsObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketInfo{}, BucketNameInvalid{Bucket: bucket}
+	}
+	vi, err := fs.storage.StatVol(bucket)
+	if err != nil {
+		return BucketInfo{}, toObjectErr(err, bucket)
+	}
+	return BucketInfo{
+		Name:    bucket,
+		Created: vi.Created,
+	}, nil
+}
+
+// ListBuckets - list buckets.
+func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
+	var bucketInfos []BucketInfo
+	vols, err := fs.storage.ListVols()
+	if err != nil {
+		return nil, toObjectErr(err)
+	}
+	for _, vol := range vols {
+		// StorageAPI can send volume names which are incompatible
+		// with buckets, handle it and skip them.
+		if !IsValidBucketName(vol.Name) {
+			continue
+		}
+		// Ignore the volume special bucket.
+		if vol.Name == minioMetaBucket {
+			continue
+		}
+		bucketInfos = append(bucketInfos, BucketInfo{
+			Name:    vol.Name,
+			Created: vol.Created,
+		})
+	}
+	sort.Sort(byBucketName(bucketInfos))
+	return bucketInfos, nil
+}
+
+// DeleteBucket - delete a bucket.
+func (fs fsObjects) DeleteBucket(bucket string) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	// Attempt to delete regular bucket.
+	if err := fs.storage.DeleteVol(bucket); err != nil {
+		return toObjectErr(err, bucket)
+	}
+	// Cleanup all the previously incomplete multiparts.
+	if err := cleanupDir(fs.storage, path.Join(minioMetaBucket, mpartMetaPrefix), bucket); err != nil && err != errVolumeNotFound {
+		return toObjectErr(err, bucket)
+	}
+	return nil
+}
+
+/// Object Operations
+
+// GetObject - get an object.
+func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer) (err error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify if object is valid.
+	if !IsValidObjectName(object) {
+		return ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+	// Offset and length cannot be negative.
+	if offset < 0 || length < 0 {
+		return toObjectErr(errUnexpected, bucket, object)
+	}
+	// Writer cannot be nil.
+	if writer == nil {
+		return toObjectErr(errUnexpected, bucket, object)
+	}
+
+	// Stat the file to get file size.
+	fi, err := fs.storage.StatFile(bucket, object)
+	if err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	// Reply back invalid range if the input offset and length fall out of range.
+	if offset > fi.Size || length > fi.Size {
+		return InvalidRange{offset, length, fi.Size}
+	}
+	// Reply if we have inputs with offset and length falling out of file size range.
+	if offset+length > fi.Size {
+		return InvalidRange{offset, length, fi.Size}
+	}
+
+	var totalLeft = length
+	bufSize := int64(readSizeV1)
+	if length > 0 && bufSize > length {
+		bufSize = length
+	}
+	// Allocate a staging buffer.
+	buf := make([]byte, int(bufSize))
+	for {
+		// Figure out the right size for the buffer.
+		curLeft := bufSize
+		if totalLeft < bufSize {
+			curLeft = totalLeft
+		}
+		// Reads the file at offset.
+		nr, er := fs.storage.ReadFile(bucket, object, offset, buf[:curLeft])
+		if nr > 0 {
+			// Write to response writer.
+			nw, ew := writer.Write(buf[0:nr])
+			if nw > 0 {
+				// Decrement whats left to write.
+				totalLeft -= int64(nw)
+
+				// Progress the offset
+				offset += int64(nw)
+			}
+			if ew != nil {
+				err = ew
+				break
+			}
+			if nr != int64(nw) {
+				err = io.ErrShortWrite
+				break
+			}
+		}
+		if er == io.EOF || er == io.ErrUnexpectedEOF {
+			break
+		}
+		if er != nil {
+			err = er
+			break
+		}
+		if totalLeft == 0 {
+			break
+		}
+	}
+	// Returns any error.
+	return toObjectErr(err, bucket, object)
+}
+
+// GetObjectInfo - get object info.
+func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return ObjectInfo{}, (BucketNameInvalid{Bucket: bucket})
+	}
+	// Verify if object is valid.
+	if !IsValidObjectName(object) {
+		return ObjectInfo{}, (ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+	fi, err := fs.storage.StatFile(bucket, object)
+	if err != nil {
+		return ObjectInfo{}, toObjectErr(err, bucket, object)
+	}
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, object))
+	if err != nil && err != errFileNotFound {
+		return ObjectInfo{}, toObjectErr(err, bucket, object)
+	}
+
+	if len(fsMeta.Meta) == 0 {
+		fsMeta.Meta = make(map[string]string)
+	}
+
+	// Guess content-type from the extension if possible.
+	if fsMeta.Meta["content-type"] == "" {
+		if objectExt := path.Ext(object); objectExt != "" {
+			if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
+				fsMeta.Meta["content-type"] = content.ContentType
+			}
+		}
+	}
+
+	// Guess content-type from the extension if possible.
+	return ObjectInfo{
+		Bucket:          bucket,
+		Name:            object,
+		ModTime:         fi.ModTime,
+		Size:            fi.Size,
+		IsDir:           fi.Mode.IsDir(),
+		MD5Sum:          fsMeta.Meta["md5Sum"],
+		ContentType:     fsMeta.Meta["content-type"],
+		ContentEncoding: fsMeta.Meta["content-encoding"],
+		UserDefined:     fsMeta.Meta,
+	}, nil
+}
+
+// PutObject - create an object.
+func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{
+			Bucket: bucket,
+			Object: object,
+		}
+	}
+	// No metadata is set, allocate a new one.
+	if metadata == nil {
+		metadata = make(map[string]string)
+	}
+
+	uniqueID := getUUID()
+
+	// Uploaded object will first be written to the temporary location which will eventually
+	// be renamed to the actual location. It is first written to the temporary location
+	// so that cleaning it up will be easy if the server goes down.
+	tempObj := path.Join(tmpMetaPrefix, uniqueID)
+
+	// Initialize md5 writer.
+	md5Writer := md5.New()
+
+	// Limit the reader to its provided size if specified.
+	var limitDataReader io.Reader
+	if size > 0 {
+		// This is done so that we can avoid erroneous clients sending more data than the set content size.
+		limitDataReader = io.LimitReader(data, size)
+	} else {
+		// else we read till EOF.
+		limitDataReader = data
+	}
+
+	if size == 0 {
+		// For size 0 we write a 0byte file.
+		err := fs.storage.AppendFile(minioMetaBucket, tempObj, []byte(""))
+		if err != nil {
+			return "", toObjectErr(err, bucket, object)
+		}
+	} else {
+		// Allocate a buffer to Read() from request body
+		bufSize := int64(readSizeV1)
+		if size > 0 && bufSize > size {
+			bufSize = size
+		}
+		buf := make([]byte, int(bufSize))
+		teeReader := io.TeeReader(limitDataReader, md5Writer)
+		bytesWritten, err := fsCreateFile(fs.storage, teeReader, buf, minioMetaBucket, tempObj)
+		if err != nil {
+			fs.storage.DeleteFile(minioMetaBucket, tempObj)
+			return "", toObjectErr(err, bucket, object)
+		}
+
+		// Should return IncompleteBody{} error when reader has fewer
+		// bytes than specified in request header.
+		if bytesWritten < size {
+			fs.storage.DeleteFile(minioMetaBucket, tempObj)
+			return "", IncompleteBody{}
+		}
+	}
+
+	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
+	// Update the md5sum if not set with the newly calculated one.
+	if len(metadata["md5Sum"]) == 0 {
+		metadata["md5Sum"] = newMD5Hex
+	}
+
+	// Validate if payload is valid.
+	if isSignVerify(data) {
+		if vErr := data.(*signVerifyReader).Verify(); vErr != nil {
+			// Incoming payload wrong, delete the temporary object.
+			fs.storage.DeleteFile(minioMetaBucket, tempObj)
+			// Error return.
+			return "", toObjectErr(vErr, bucket, object)
+		}
+	}
+
+	// md5Hex representation.
+	md5Hex := metadata["md5Sum"]
+	if md5Hex != "" {
+		if newMD5Hex != md5Hex {
+			// MD5 mismatch, delete the temporary object.
+			fs.storage.DeleteFile(minioMetaBucket, tempObj)
+			// Returns md5 mismatch.
+			return "", BadDigest{md5Hex, newMD5Hex}
+		}
+	}
+
+	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
+	err := fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
+	if err != nil {
+		return "", toObjectErr(err, bucket, object)
+	}
+
+	// Save additional metadata only if extended headers such as "X-Amz-Meta-" are set.
+	if hasExtendedHeader(metadata) {
+		// Initialize `fs.json` values.
+		fsMeta := newFSMetaV1()
+		fsMeta.Meta = metadata
+
+		// Write the metadata to a temp file and rename it to the actual location.
+		tmpMetaPath := path.Join(tmpMetaPrefix, getUUID())
+		fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
+		if err = writeFSMetadata(fs.storage, minioMetaBucket, tmpMetaPath, fsMeta); err != nil {
+			return "", toObjectErr(err, bucket, object)
+		}
+		if err = fs.storage.RenameFile(minioMetaBucket, tmpMetaPath, minioMetaBucket, fsMetaPath); err != nil {
+			return "", toObjectErr(err, bucket, object)
+		}
+	}
+
+	// Return md5sum, successfully wrote object.
+	return newMD5Hex, nil
+}
+
+// DeleteObject - deletes an object from a bucket, this operation is destructive
+// and there are no rollbacks supported.
+func (fs fsObjects) DeleteObject(bucket, object string) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+	if !IsValidObjectName(object) {
+		return ObjectNameInvalid{Bucket: bucket, Object: object}
+	}
+	err := fs.storage.DeleteFile(minioMetaBucket, path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile))
+	if err != nil && err != errFileNotFound {
+		return toObjectErr(err, bucket, object)
+	}
+	if err = fs.storage.DeleteFile(bucket, object); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+	return nil
+}
+
+// Checks whether bucket exists.
+func isBucketExist(storage StorageAPI, bucketName string) bool {
+	// Check whether bucket exists.
+	_, err := storage.StatVol(bucketName)
+	if err != nil {
+		if err == errVolumeNotFound {
+			return false
+		}
+		errorIf(err, "Stat failed on bucket "+bucketName+".")
+		return false
+	}
+	return true
+}
+
+// ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
+// state for future re-entrant list requests.
+func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+	// Convert entry to FileInfo
+	entryToFileInfo := func(entry string) (fileInfo FileInfo, err error) {
+		if strings.HasSuffix(entry, slashSeparator) {
+			// Object name needs to be full path.
+			fileInfo.Name = entry
+			fileInfo.Mode = os.ModeDir
+			return
+		}
+		if fileInfo, err = fs.storage.StatFile(bucket, entry); err != nil {
+			return
+		}
+		fsMeta, mErr := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, entry))
+		if mErr != nil && mErr != errFileNotFound {
+			return FileInfo{}, mErr
+		}
+		if len(fsMeta.Meta) == 0 {
+			fsMeta.Meta = make(map[string]string)
+		}
+		// Object name needs to be full path.
+		fileInfo.Name = entry
+		fileInfo.MD5Sum = fsMeta.Meta["md5Sum"]
+		return
+	}
+
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return ListObjectsInfo{}, BucketNameInvalid{Bucket: bucket}
+	}
+	// Verify if bucket exists.
+	if !isBucketExist(fs.storage, bucket) {
+		return ListObjectsInfo{}, BucketNotFound{Bucket: bucket}
+	}
+	if !IsValidObjectPrefix(prefix) {
+		return ListObjectsInfo{}, ObjectNameInvalid{Bucket: bucket, Object: prefix}
+	}
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != slashSeparator {
+		return ListObjectsInfo{}, UnsupportedDelimiter{
+			Delimiter: delimiter,
+		}
+	}
+	// Verify if marker has prefix.
+	if marker != "" {
+		if !strings.HasPrefix(marker, prefix) {
+			return ListObjectsInfo{}, InvalidMarkerPrefixCombination{
+				Marker: marker,
+				Prefix: prefix,
+			}
+		}
+	}
+
+	// With max keys of zero we have reached eof, return right here.
+	if maxKeys == 0 {
+		return ListObjectsInfo{}, nil
+	}
+
+	// For delimiter and prefix as '/' we do not list anything at all
+	// since according to s3 spec we stop at the 'delimiter'
+	// along // with the prefix. On a flat namespace with 'prefix'
+	// as '/' we don't have any entries, since all the keys are
+	// of form 'keyName/...'
+	if delimiter == slashSeparator && prefix == slashSeparator {
+		return ListObjectsInfo{}, nil
+	}
+
+	// Over flowing count - reset to maxObjectList.
+	if maxKeys < 0 || maxKeys > maxObjectList {
+		maxKeys = maxObjectList
+	}
+
+	// Default is recursive, if delimiter is set then list non recursive.
+	recursive := true
+	if delimiter == slashSeparator {
+		recursive = false
+	}
+
+	heal := false // true only for xl.ListObjectsHeal()
+	walkResultCh, endWalkCh := fs.listPool.Release(listParams{bucket, recursive, marker, prefix, heal})
+	if walkResultCh == nil {
+		endWalkCh = make(chan struct{})
+		isLeaf := func(bucket, object string) bool {
+			// bucket argument is unused as we don't need to StatFile
+			// to figure if it's a file, just need to check that the
+			// object string does not end with "/".
+			return !strings.HasSuffix(object, slashSeparator)
+		}
+		listDir := listDirFactory(isLeaf, fs.storage)
+		walkResultCh = startTreeWalk(bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh)
+	}
+	var fileInfos []FileInfo
+	var eof bool
+	var nextMarker string
+	for i := 0; i < maxKeys; {
+		walkResult, ok := <-walkResultCh
+		if !ok {
+			// Closed channel.
+			eof = true
+			break
+		}
+		// For any walk error return right away.
+		if walkResult.err != nil {
+			// File not found is a valid case.
+			if walkResult.err == errFileNotFound {
+				return ListObjectsInfo{}, nil
+			}
+			return ListObjectsInfo{}, toObjectErr(walkResult.err, bucket, prefix)
+		}
+		fileInfo, err := entryToFileInfo(walkResult.entry)
+		if err != nil {
+			return ListObjectsInfo{}, nil
+		}
+		nextMarker = fileInfo.Name
+		fileInfos = append(fileInfos, fileInfo)
+		if walkResult.end {
+			eof = true
+			break
+		}
+		i++
+	}
+	params := listParams{bucket, recursive, nextMarker, prefix, heal}
+	if !eof {
+		fs.listPool.Set(params, walkResultCh, endWalkCh)
+	}
+
+	result := ListObjectsInfo{IsTruncated: !eof}
+	for _, fileInfo := range fileInfos {
+		result.NextMarker = fileInfo.Name
+		if fileInfo.Mode.IsDir() {
+			result.Prefixes = append(result.Prefixes, fileInfo.Name)
+			continue
+		}
+		result.Objects = append(result.Objects, ObjectInfo{
+			Name:    fileInfo.Name,
+			ModTime: fileInfo.ModTime,
+			Size:    fileInfo.Size,
+			MD5Sum:  fileInfo.MD5Sum,
+			IsDir:   false,
+		})
+	}
+	return result, nil
+}
+
+// HealObject - no-op for fs. Valid only for XL.
+func (fs fsObjects) HealObject(bucket, object string) error {
+	return NotImplemented{}
+}
+
+// HealListObjects - list objects for healing. Valid only for XL
+func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+	return ListObjectsInfo{}, NotImplemented{}
+}

cmd/fs-v1_test.go

@@ -0,0 +1,61 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestNewFS - tests initialization of all input disks
+// and constructs a valid `FS` object layer.
+func TestNewFS(t *testing.T) {
+	// Do not attempt to create this path, the test validates
+	// so that newFSObjects initializes non existing paths
+	// and successfully returns initialized object layer.
+	disk := filepath.Join(os.TempDir(), "minio-"+nextSuffix())
+	defer removeAll(disk)
+
+	// Setup to test errFSDiskFormat.
+	disks := []string{}
+	for i := 0; i < 6; i++ {
+		xlDisk := filepath.Join(os.TempDir(), "minio-"+nextSuffix())
+		defer removeAll(xlDisk)
+		disks = append(disks, xlDisk)
+	}
+
+	// Initializes all disks with XL
+	_, err := newXLObjects(disks, nil)
+	if err != nil {
+		t.Fatalf("Unable to initialize XL object, %s", err)
+	}
+
+	testCases := []struct {
+		disk        string
+		expectedErr error
+	}{
+		{disk, nil},
+		{disks[0], errFSDiskFormat},
+	}
+
+	for _, testCase := range testCases {
+		if _, err := newFSObjects(testCase.disk); err != testCase.expectedErr {
+			t.Fatalf("expected: %s, got: %s", testCase.expectedErr, err)
+		}
+	}
+}

cmd/generic-handlers.go

@@ -0,0 +1,314 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	router "github.com/gorilla/mux"
+	"github.com/rs/cors"
+)
+
+// HandlerFunc - useful to chain different middleware http.Handler
+type HandlerFunc func(http.Handler) http.Handler
+
+func registerHandlers(mux *router.Router, handlerFns ...HandlerFunc) http.Handler {
+	var f http.Handler
+	f = mux
+	for _, hFn := range handlerFns {
+		f = hFn(f)
+	}
+	return f
+}
+
+// Adds limiting body size middleware
+
+// Set the body size limit to 6 Gb = Maximum object size + other possible data
+// in the same request
+const requestMaxBodySize = 1024 * 1024 * 1024 * (5 + 1)
+
+type requestSizeLimitHandler struct {
+	handler     http.Handler
+	maxBodySize int64
+}
+
+func setRequestSizeLimitHandler(h http.Handler) http.Handler {
+	return requestSizeLimitHandler{handler: h, maxBodySize: requestMaxBodySize}
+}
+
+func (h requestSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Restricting read data to a given maximum length
+	r.Body = http.MaxBytesReader(w, r.Body, h.maxBodySize)
+	h.handler.ServeHTTP(w, r)
+}
+
+// Adds redirect rules for incoming requests.
+type redirectHandler struct {
+	handler        http.Handler
+	locationPrefix string
+}
+
+// Reserved bucket.
+const (
+	reservedBucket = "/minio"
+)
+
+func setBrowserRedirectHandler(h http.Handler) http.Handler {
+	return redirectHandler{handler: h, locationPrefix: reservedBucket}
+}
+
+func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Re-direction handled specifically for browsers.
+	if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && !isRequestSignatureV4(r) {
+		// '/' is redirected to 'locationPrefix/'
+		// '/webrpc' is redirected to 'locationPrefix/webrpc'
+		// '/login' is redirected to 'locationPrefix/login'
+		switch r.URL.Path {
+		case "/", "/webrpc", "/login", "/favicon.ico":
+			location := h.locationPrefix + r.URL.Path
+			// Redirect to new location.
+			http.Redirect(w, r, location, http.StatusTemporaryRedirect)
+			return
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// Adds Cache-Control header
+type cacheControlHandler struct {
+	handler http.Handler
+}
+
+func setBrowserCacheControlHandler(h http.Handler) http.Handler {
+	return cacheControlHandler{h}
+}
+
+func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method == "GET" && strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
+		// For all browser requests set appropriate Cache-Control policies
+		match, e := regexp.MatchString(reservedBucket+`/([^/]+\.js|favicon.ico)`, r.URL.Path)
+		if e != nil {
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		if match {
+			// For assets set cache expiry of one year. For each release, the name
+			// of the asset name will change and hence it can not be served from cache.
+			w.Header().Set("Cache-Control", "max-age=31536000")
+		} else if strings.HasPrefix(r.URL.Path, reservedBucket+"/") {
+			// For non asset requests we serve index.html which will never be cached.
+			w.Header().Set("Cache-Control", "no-store")
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// Adds verification for incoming paths.
+type minioPrivateBucketHandler struct {
+	handler        http.Handler
+	reservedBucket string
+}
+
+func setPrivateBucketHandler(h http.Handler) http.Handler {
+	return minioPrivateBucketHandler{handler: h, reservedBucket: reservedBucket}
+}
+
+func (h minioPrivateBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// For all non browser requests, reject access to 'reservedBucket'.
+	if !strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && path.Clean(r.URL.Path) == reservedBucket {
+		writeErrorResponse(w, r, ErrAllAccessDisabled, r.URL.Path)
+		return
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// Supported Amz date formats.
+var amzDateFormats = []string{
+	time.RFC1123,
+	time.RFC1123Z,
+	iso8601Format,
+	// Add new AMZ date formats here.
+}
+
+// parseAmzDate - parses date string into supported amz date formats.
+func parseAmzDate(amzDateStr string) (amzDate time.Time, apiErr APIErrorCode) {
+	for _, dateFormat := range amzDateFormats {
+		amzDate, e := time.Parse(dateFormat, amzDateStr)
+		if e == nil {
+			return amzDate, ErrNone
+		}
+	}
+	return time.Time{}, ErrMalformedDate
+}
+
+// Supported Amz date headers.
+var amzDateHeaders = []string{
+	"x-amz-date",
+	"date",
+}
+
+// parseAmzDateHeader - parses supported amz date headers, in
+// supported amz date formats.
+func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
+	for _, amzDateHeader := range amzDateHeaders {
+		amzDateStr := req.Header.Get(http.CanonicalHeaderKey(amzDateHeader))
+		if amzDateStr != "" {
+			return parseAmzDate(amzDateStr)
+		}
+	}
+	// Date header missing.
+	return time.Time{}, ErrMissingDateHeader
+}
+
+type timeHandler struct {
+	handler http.Handler
+}
+
+// setTimeValidityHandler to validate parsable time over http header
+func setTimeValidityHandler(h http.Handler) http.Handler {
+	return timeHandler{h}
+}
+
+func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Verify if date headers are set, if not reject the request
+	if _, ok := r.Header["Authorization"]; ok {
+		amzDate, apiErr := parseAmzDateHeader(r)
+		if apiErr != ErrNone {
+			// All our internal APIs are sensitive towards Date
+			// header, for all requests where Date header is not
+			// present we will reject such clients.
+			writeErrorResponse(w, r, apiErr, r.URL.Path)
+			return
+		}
+		// Verify if the request date header is more than 5minutes
+		// late, reject such clients.
+		if time.Now().UTC().Sub(amzDate)/time.Minute > time.Duration(5)*time.Minute {
+			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
+			return
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+type resourceHandler struct {
+	handler http.Handler
+}
+
+// setCorsHandler handler for CORS (Cross Origin Resource Sharing)
+func setCorsHandler(h http.Handler) http.Handler {
+	c := cors.New(cors.Options{
+		AllowedOrigins: []string{"*"},
+		AllowedMethods: []string{"GET", "HEAD", "POST", "PUT"},
+		AllowedHeaders: []string{"*"},
+		ExposedHeaders: []string{"ETag"},
+	})
+	return c.Handler(h)
+}
+
+// setIgnoreResourcesHandler -
+// Ignore resources handler is wrapper handler used for API request resource validation
+// Since we do not support all the S3 queries, it is necessary for us to throw back a
+// valid error message indicating that requested feature is not implemented.
+func setIgnoreResourcesHandler(h http.Handler) http.Handler {
+	return resourceHandler{h}
+}
+
+// Resource handler ServeHTTP() wrapper
+func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Skip the first element which is usually '/' and split the rest.
+	splits := strings.SplitN(r.URL.Path[1:], "/", 2)
+
+	// Save bucketName and objectName extracted from url Path.
+	var bucketName, objectName string
+	if len(splits) == 1 {
+		bucketName = splits[0]
+	}
+	if len(splits) == 2 {
+		bucketName = splits[0]
+		objectName = splits[1]
+	}
+
+	// If bucketName is present and not objectName check for bucket level resource queries.
+	if bucketName != "" && objectName == "" {
+		if ignoreNotImplementedBucketResources(r) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
+	}
+	// If bucketName and objectName are present check for its resource queries.
+	if bucketName != "" && objectName != "" {
+		if ignoreNotImplementedObjectResources(r) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
+	}
+	// A put method on path "/" doesn't make sense, ignore it.
+	if r.Method == "PUT" && r.URL.Path == "/" {
+		writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+		return
+	}
+
+	// Serve HTTP.
+	h.handler.ServeHTTP(w, r)
+}
+
+//// helpers
+
+// Checks requests for not implemented Bucket resources
+func ignoreNotImplementedBucketResources(req *http.Request) bool {
+	for name := range req.URL.Query() {
+		if notimplementedBucketResourceNames[name] {
+			return true
+		}
+	}
+	return false
+}
+
+// Checks requests for not implemented Object resources
+func ignoreNotImplementedObjectResources(req *http.Request) bool {
+	for name := range req.URL.Query() {
+		if notimplementedObjectResourceNames[name] {
+			return true
+		}
+	}
+	return false
+}
+
+// List of not implemented bucket queries
+var notimplementedBucketResourceNames = map[string]bool{
+	"acl":            true,
+	"cors":           true,
+	"lifecycle":      true,
+	"logging":        true,
+	"replication":    true,
+	"tagging":        true,
+	"versions":       true,
+	"requestPayment": true,
+	"versioning":     true,
+	"website":        true,
+}
+
+// List of not implemented object queries
+var notimplementedObjectResourceNames = map[string]bool{
+	"torrent": true,
+	"acl":     true,
+	"policy":  true,
+}

cmd/globals.go

@@ -0,0 +1,65 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"github.com/fatih/color"
+	"github.com/minio/minio/pkg/objcache"
+)
+
+// Global constants for Minio.
+const (
+	minGoVersion = ">= 1.6" // Minio requires at least Go v1.6
+)
+
+// minio configuration related constants.
+const (
+	globalMinioConfigVersion = "6"
+	globalMinioConfigDir     = ".minio"
+	globalMinioCertsDir      = "certs"
+	globalMinioCertFile      = "public.crt"
+	globalMinioKeyFile       = "private.key"
+	globalMinioConfigFile    = "config.json"
+	// Add new global values here.
+)
+
+var (
+	globalQuiet = false // Quiet flag set via command line
+	globalTrace = false // Trace flag set via environment setting.
+	// Add new global flags here.
+
+	// Maximum connections handled per
+	// server, defaults to 0 (unlimited).
+	globalMaxConn = 0
+	// Maximum cache size.
+	globalMaxCacheSize = uint64(maxCacheSize)
+	// Cache expiry.
+	globalCacheExpiry = objcache.DefaultExpiry
+	// Add new variable global values here.
+)
+
+var (
+	// Limit fields size (except file) to 1Mib since Policy document
+	// can reach that size according to https://aws.amazon.com/articles/1434
+	maxFormFieldSize = int64(1024 * 1024)
+)
+
+// global colors.
+var (
+	colorBlue = color.New(color.FgBlue).SprintfFunc()
+	colorBold = color.New(color.Bold).SprintFunc()
+)

cmd/handler-utils.go

@@ -0,0 +1,145 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Validates location constraint in PutBucket request body.
+// The location value in the request body should match the
+// region configured at serverConfig, otherwise error is returned.
+func isValidLocationConstraint(r *http.Request) (s3Error APIErrorCode) {
+	serverRegion := serverConfig.GetRegion()
+	// If the request has no body with content-length set to 0,
+	// we do not have to validate location constraint. Bucket will
+	// be created at default region.
+	if r.ContentLength == 0 {
+		return ErrNone
+	}
+	locationConstraint := createBucketLocationConfiguration{}
+	if err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength); err != nil {
+		if err == io.EOF && r.ContentLength == -1 {
+			// EOF is a valid condition here when ContentLength is -1.
+			return ErrNone
+		}
+		errorIf(err, "Unable to xml decode location constraint")
+		// Treat all other failures as XML parsing errors.
+		return ErrMalformedXML
+	} // Successfully decoded, proceed to verify the region.
+
+	// Once region has been obtained we proceed to verify it.
+	incomingRegion := locationConstraint.Location
+	if incomingRegion == "" {
+		// Location constraint is empty for region "us-east-1",
+		// in accordance with protocol.
+		incomingRegion = "us-east-1"
+	}
+	// Return errInvalidRegion if location constraint does not match
+	// with configured region.
+	s3Error = ErrNone
+	if serverRegion != incomingRegion {
+		s3Error = ErrInvalidRegion
+	}
+	return s3Error
+}
+
+// Supported headers that needs to be extracted.
+var supportedHeaders = []string{
+	"content-type",
+	"cache-control",
+	"content-encoding",
+	"content-disposition",
+	// Add more supported headers here.
+}
+
+// extractMetadataFromHeader extracts metadata from HTTP header.
+func extractMetadataFromHeader(header http.Header) map[string]string {
+	metadata := make(map[string]string)
+	// Save standard supported headers.
+	for _, supportedHeader := range supportedHeaders {
+		canonicalHeader := http.CanonicalHeaderKey(supportedHeader)
+		// HTTP headers are case insensitive, look for both canonical
+		// and non canonical entries.
+		if _, ok := header[canonicalHeader]; ok {
+			metadata[supportedHeader] = header.Get(canonicalHeader)
+		} else if _, ok := header[supportedHeader]; ok {
+			metadata[supportedHeader] = header.Get(supportedHeader)
+		}
+	}
+	// Go through all other headers for any additional headers that needs to be saved.
+	for key := range header {
+		cKey := http.CanonicalHeaderKey(key)
+		if strings.HasPrefix(cKey, "X-Amz-Meta-") {
+			metadata[cKey] = header.Get(cKey)
+		} else if strings.HasPrefix(key, "X-Minio-Meta-") {
+			metadata[cKey] = header.Get(cKey)
+		}
+	}
+	// Return.
+	return metadata
+}
+
+// Extract form fields and file data from a HTTP POST Policy
+func extractPostPolicyFormValues(reader *multipart.Reader) (filePart io.Reader, fileName string, formValues map[string]string, err error) {
+	/// HTML Form values
+	formValues = make(map[string]string)
+	fileName = ""
+	for err == nil {
+		var part *multipart.Part
+		part, err = reader.NextPart()
+		if part != nil {
+			canonicalFormName := http.CanonicalHeaderKey(part.FormName())
+			if canonicalFormName != "File" {
+				var buffer []byte
+				limitReader := io.LimitReader(part, maxFormFieldSize+1)
+				buffer, err = ioutil.ReadAll(limitReader)
+				if err != nil {
+					return nil, "", nil, err
+				}
+				if int64(len(buffer)) > maxFormFieldSize {
+					return nil, "", nil, errSizeUnexpected
+				}
+				formValues[canonicalFormName] = string(buffer)
+			} else {
+				filePart = io.LimitReader(part, maxObjectSize)
+				fileName = part.FileName()
+				// As described in S3 spec, we expect file to be the last form field
+				break
+			}
+		}
+	}
+	return filePart, fileName, formValues, nil
+}
+
+// Send whitespace character, once every 5secs, until CompleteMultipartUpload is done.
+// CompleteMultipartUpload method of the object layer indicates that it's done via doneCh
+func sendWhiteSpaceChars(w http.ResponseWriter, doneCh <-chan struct{}) {
+	for {
+		select {
+		case <-time.After(5 * time.Second):
+			w.Write([]byte(" "))
+		case <-doneCh:
+			return
+		}
+	}
+}

cmd/handler-utils_test.go

@@ -0,0 +1,108 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/xml"
+	"io/ioutil"
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+// Tests validate bucket LocationConstraint.
+func TestIsValidLocationContraint(t *testing.T) {
+	path, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer removeAll(path)
+
+	// generates the input request with XML bucket configuration set to the request body.
+	createExpectedRequest := func(req *http.Request, location string) (*http.Request, error) {
+		createBucketConfig := createBucketLocationConfiguration{}
+		createBucketConfig.Location = location
+		var createBucketConfigBytes []byte
+		createBucketConfigBytes, e := xml.Marshal(createBucketConfig)
+		if e != nil {
+			return nil, e
+		}
+		createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes)
+		req.Body = ioutil.NopCloser(createBucketConfigBuffer)
+		req.ContentLength = int64(createBucketConfigBuffer.Len())
+		return req, nil
+	}
+
+	testCases := []struct {
+		locationForInputRequest string
+		serverConfigRegion      string
+		expectedCode            APIErrorCode
+	}{
+		// Test case - 1.
+		{"us-east-1", "us-east-1", ErrNone},
+		// Test case - 2.
+		// In case of empty request body ErrNone is returned.
+		{"", "us-east-1", ErrNone},
+		// Test case - 3.
+		{"eu-central-1", "us-east-1", ErrInvalidRegion},
+	}
+	for i, testCase := range testCases {
+		inputRequest, e := createExpectedRequest(&http.Request{}, testCase.locationForInputRequest)
+		if e != nil {
+			t.Fatalf("Test %d: Failed to Marshal bucket configuration", i+1)
+		}
+		serverConfig.SetRegion(testCase.serverConfigRegion)
+		actualCode := isValidLocationConstraint(inputRequest)
+		if testCase.expectedCode != actualCode {
+			t.Errorf("Test %d: Expected the APIErrCode to be %d, but instead found %d", i+1, testCase.expectedCode, actualCode)
+		}
+	}
+}
+
+// Tests validate metadata extraction from http headers.
+func TestExtractMetadataHeaders(t *testing.T) {
+	testCases := []struct {
+		header   http.Header
+		metadata map[string]string
+	}{
+		// Validate if there a known 'content-type'.
+		{
+			header: http.Header{
+				"Content-Type": []string{"image/png"},
+			},
+			metadata: map[string]string{
+				"content-type": "image/png",
+			},
+		},
+		// Validate if there are no keys to extract.
+		{
+			header: http.Header{
+				"test-1": []string{"123"},
+			},
+			metadata: map[string]string{},
+		},
+	}
+
+	// Validate if the extracting headers.
+	for i, testCase := range testCases {
+		metadata := extractMetadataFromHeader(testCase.header)
+		if !reflect.DeepEqual(metadata, testCase.metadata) {
+			t.Fatalf("Test %d failed: Expected \"%#v\", got \"%#v\"", i+1, testCase.metadata, metadata)
+		}
+	}
+}

cmd/host-to-ip.go

@@ -0,0 +1,42 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net"
+	"sort"
+)
+
+// byLastOctet implements sort.Interface used in sorting a list
+// of ip address by their last octet value.
+type byLastOctet []net.IP
+
+func (n byLastOctet) Len() int      { return len(n) }
+func (n byLastOctet) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
+func (n byLastOctet) Less(i, j int) bool {
+	return []byte(n[i].To4())[3] < []byte(n[j].To4())[3]
+}
+
+// getIPsFromHosts - returns a reverse sorted list of ips based on the last octet value.
+func getIPsFromHosts(hosts []string) (ips []net.IP) {
+	for _, host := range hosts {
+		ips = append(ips, net.ParseIP(host))
+	}
+	// Reverse sort ips by their last octet.
+	sort.Sort(sort.Reverse(byLastOctet(ips)))
+	return ips
+}

cmd/host-to-ip_test.go

@@ -0,0 +1,61 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "testing"
+
+// Tests sorted list generated from hosts to ip.
+func TestHostToIP(t *testing.T) {
+	// Collection of test cases to validate last octet sorting.
+	testCases := []struct {
+		hosts       []string
+		sortedHosts []string
+	}{
+		{
+			// List of ip addresses that need to be sorted.
+			[]string{
+				"129.95.30.40",
+				"5.24.69.2",
+				"19.20.203.5",
+				"1.2.3.4",
+				"127.0.0.1",
+				"19.20.21.22",
+				"5.220.100.50",
+			},
+			// Numerical sorting result based on the last octet.
+			[]string{
+				"5.220.100.50",
+				"129.95.30.40",
+				"19.20.21.22",
+				"19.20.203.5",
+				"1.2.3.4",
+				"5.24.69.2",
+				"127.0.0.1",
+			},
+		},
+	}
+
+	// Tests the correct sorting behavior of getIPsFromHosts.
+	for j, testCase := range testCases {
+		ips := getIPsFromHosts(testCase.hosts)
+		for i, ip := range ips {
+			if ip.String() != testCase.sortedHosts[i] {
+				t.Fatalf("Test %d expected to pass but failed. Wanted ip %s, but got %s", j+1, testCase.sortedHosts[i], ip.String())
+			}
+		}
+	}
+}

cmd/httprange.go

@@ -0,0 +1,134 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+const (
+	byteRangePrefix = "bytes="
+)
+
+// Valid byte position regexp
+var validBytePos = regexp.MustCompile(`^[0-9]+$`)
+
+// HttpRange specifies the byte range to be sent to the client.
+type httpRange struct {
+	offsetBegin  int64
+	offsetEnd    int64
+	resourceSize int64
+}
+
+// String populate range stringer interface
+func (hrange httpRange) String() string {
+	return fmt.Sprintf("bytes %d-%d/%d", hrange.offsetBegin, hrange.offsetEnd, hrange.resourceSize)
+}
+
+// getlength - get length from the range.
+func (hrange httpRange) getLength() int64 {
+	return 1 + hrange.offsetEnd - hrange.offsetBegin
+}
+
+func parseRequestRange(rangeString string, resourceSize int64) (hrange *httpRange, err error) {
+	// Return error if given range string doesn't start with byte range prefix.
+	if !strings.HasPrefix(rangeString, byteRangePrefix) {
+		return nil, fmt.Errorf("'%s' does not start with '%s'", rangeString, byteRangePrefix)
+	}
+
+	// Trim byte range prefix.
+	byteRangeString := strings.TrimPrefix(rangeString, byteRangePrefix)
+
+	// Check if range string contains delimiter '-', else return error. eg. "bytes=8"
+	sepIndex := strings.Index(byteRangeString, "-")
+	if sepIndex == -1 {
+		return nil, fmt.Errorf("'%s' does not have a valid range value", rangeString)
+	}
+
+	offsetBeginString := byteRangeString[:sepIndex]
+	offsetBegin := int64(-1)
+	// Convert offsetBeginString only if its not empty.
+	if len(offsetBeginString) > 0 {
+		if !validBytePos.MatchString(offsetBeginString) {
+			return nil, fmt.Errorf("'%s' does not have a valid first byte position value", rangeString)
+		}
+
+		if offsetBegin, err = strconv.ParseInt(offsetBeginString, 10, 64); err != nil {
+			return nil, fmt.Errorf("'%s' does not have a valid first byte position value", rangeString)
+		}
+	}
+
+	offsetEndString := byteRangeString[sepIndex+1:]
+	offsetEnd := int64(-1)
+	// Convert offsetEndString only if its not empty.
+	if len(offsetEndString) > 0 {
+		if !validBytePos.MatchString(offsetEndString) {
+			return nil, fmt.Errorf("'%s' does not have a valid last byte position value", rangeString)
+		}
+
+		if offsetEnd, err = strconv.ParseInt(offsetEndString, 10, 64); err != nil {
+			return nil, fmt.Errorf("'%s' does not have a valid last byte position value", rangeString)
+		}
+	}
+
+	// rangeString contains first and last byte positions. eg. "bytes=2-5"
+	if offsetBegin > -1 && offsetEnd > -1 {
+		if offsetBegin > offsetEnd {
+			// Last byte position is not greater than first byte position. eg. "bytes=5-2"
+			return nil, fmt.Errorf("'%s' does not have valid range value", rangeString)
+		}
+
+		// First and last byte positions should not be >= resourceSize.
+		if offsetBegin >= resourceSize {
+			return nil, errInvalidRange
+		}
+
+		if offsetEnd >= resourceSize {
+			offsetEnd = resourceSize - 1
+		}
+	} else if offsetBegin > -1 {
+		// rangeString contains only first byte position. eg. "bytes=8-"
+		if offsetBegin >= resourceSize {
+			// First byte position should not be >= resourceSize.
+			return nil, errInvalidRange
+		}
+
+		offsetEnd = resourceSize - 1
+	} else if offsetEnd > -1 {
+		// rangeString contains only last byte position. eg. "bytes=-3"
+		if offsetEnd == 0 {
+			// Last byte position should not be zero eg. "bytes=-0"
+			return nil, errInvalidRange
+		}
+
+		if offsetEnd >= resourceSize {
+			offsetBegin = 0
+		} else {
+			offsetBegin = resourceSize - offsetEnd
+		}
+
+		offsetEnd = resourceSize - 1
+	} else {
+		// rangeString contains first and last byte positions missing. eg. "bytes=-"
+		return nil, fmt.Errorf("'%s' does not have valid range value", rangeString)
+	}
+
+	return &httpRange{offsetBegin, offsetEnd, resourceSize}, nil
+}

cmd/httprange_test.go

@@ -0,0 +1,90 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "testing"
+
+// Test parseRequestRange()
+func TestParseRequestRange(t *testing.T) {
+	// Test success cases.
+	successCases := []struct {
+		rangeString string
+		offsetBegin int64
+		offsetEnd   int64
+		length      int64
+	}{
+		{"bytes=2-5", 2, 5, 4},
+		{"bytes=2-20", 2, 9, 8},
+		{"bytes=2-2", 2, 2, 1},
+		{"bytes=0000-0006", 0, 6, 7},
+		{"bytes=2-", 2, 9, 8},
+		{"bytes=-4", 6, 9, 4},
+		{"bytes=-20", 0, 9, 10},
+	}
+
+	for _, successCase := range successCases {
+		hrange, err := parseRequestRange(successCase.rangeString, 10)
+		if err != nil {
+			t.Fatalf("expected: <nil>, got: %s", err)
+		}
+
+		if hrange.offsetBegin != successCase.offsetBegin {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, hrange.offsetBegin)
+		}
+
+		if hrange.offsetEnd != successCase.offsetEnd {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, hrange.offsetEnd)
+		}
+		if hrange.getLength() != successCase.length {
+			t.Fatalf("expected: %d, got: %d", successCase.length, hrange.getLength())
+		}
+	}
+
+	// Test invalid range strings.
+	invalidRangeStrings := []string{
+		"bytes=8",
+		"bytes=5-2",
+		"bytes=+2-5",
+		"bytes=2-+5",
+		"bytes=2--5",
+		"bytes=-",
+		"",
+		"2-5",
+		"bytes = 2-5",
+		"bytes=2 - 5",
+		"bytes=0-0,-1",
+		"bytes=2-5 ",
+	}
+	for _, rangeString := range invalidRangeStrings {
+		if _, err := parseRequestRange(rangeString, 10); err == nil {
+			t.Fatalf("expected: an error, got: <nil>")
+		}
+	}
+
+	// Test error range strings.
+	errorRangeString := []string{
+		"bytes=10-10",
+		"bytes=20-30",
+		"bytes=20-",
+		"bytes=-0",
+	}
+	for _, rangeString := range errorRangeString {
+		if _, err := parseRequestRange(rangeString, 10); err != errInvalidRange {
+			t.Fatalf("expected: %s, got: %s", errInvalidRange, err)
+		}
+	}
+}

cmd/leak-detect_test.go

@@ -0,0 +1,154 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package leaktest provides tools to detect leaked goroutines in tests.
+// To use it, call "defer leaktest.AfterTest(t)()" at the beginning of each
+// test that may use goroutines.
+package cmd
+
+import (
+	"runtime"
+	"sort"
+	"strings"
+	"time"
+)
+
+const (
+	// deadline (in seconds) up to which the go routine leak detection has to be retried.
+	leakDetectDeadline = 5
+	// pause time (in milliseconds) between each snapshot at the end of the go routine leak detection.
+	leakDetectPauseTimeMs = 50
+)
+
+// LeakDetect - type with  methods for go routine leak detection.
+type LeakDetect struct {
+	relevantRoutines map[string]bool
+}
+
+// NewLeakDetect - Initialize a LeakDetector with the snapshot of relevant Go routines.
+func NewLeakDetect() LeakDetect {
+	snapshot := LeakDetect{
+		relevantRoutines: make(map[string]bool),
+	}
+	for _, g := range pickRelevantGoroutines() {
+		snapshot.relevantRoutines[g] = true
+	}
+	return snapshot
+}
+
+// CompareCurrentSnapshot - Compares the initial relevant stack trace with the current one (during the time of invocation).
+func (initialSnapShot LeakDetect) CompareCurrentSnapshot() []string {
+	var stackDiff []string
+	for _, g := range pickRelevantGoroutines() {
+		// Identify the Go routines those were not present in the initial snapshot.
+		// In other words a stack diff.
+		if !initialSnapShot.relevantRoutines[g] {
+			stackDiff = append(stackDiff, g)
+		}
+	}
+	return stackDiff
+}
+
+// DetectLeak - Creates a snapshot of runtiem stack and compares it with  the initial stack snapshot.
+func (initialSnapShot LeakDetect) DetectLeak(t TestErrHandler) {
+	if t.Failed() {
+		return
+	}
+	// Loop, waiting for goroutines to shut down.
+	// Wait up to 5 seconds, but finish as quickly as possible.
+	deadline := time.Now().Add(leakDetectDeadline * time.Second)
+	for {
+		// get sack snapshot of relevant go routines.
+		leaked := initialSnapShot.CompareCurrentSnapshot()
+		// current stack snapshot matches the initial one, no leaks, return.
+		if len(leaked) == 0 {
+			return
+		}
+		// wait a test again will deadline.
+		if time.Now().Before(deadline) {
+			time.Sleep(leakDetectPauseTimeMs * time.Millisecond)
+			continue
+		}
+		// after the deadline time report all the difference in the latest snapshot compared with the initial one.
+		for _, g := range leaked {
+			t.Errorf("Leaked goroutine: %v", g)
+		}
+		return
+	}
+}
+
+// DetectTestLeak -  snapshots the currently-running goroutines and returns a
+// function to be run at the end of tests to see whether any
+// goroutines leaked.
+// Usage: `defer DetectTestLeak(t)()` in beginning line of benchmarks or unit tests.
+func DetectTestLeak(t TestErrHandler) func() {
+	initialStackSnapShot := NewLeakDetect()
+	return func() {
+		initialStackSnapShot.DetectLeak(t)
+	}
+}
+
+// list of functions to be ignored from the stack trace.
+// Leak detection is done when tests are run, should ignore the tests related functions,
+// and other runtime functions while identifying leaks.
+var ignoredStackFns = []string{
+	"",
+	// Below are the stacks ignored by the upstream leaktest code.
+	"testing.Main(",
+	"testing.tRunner(",
+	"testing.tRunner(",
+	"runtime.goexit",
+	"created by runtime.gc",
+	// ignore the snapshot function.
+	// since the snapshot is taken here the entry will have the current function too.
+	"pickRelevantGoroutines",
+	"runtime.MHeap_Scavenger",
+	"signal.signal_recv",
+	"sigterm.handler",
+	"runtime_mcall",
+	"goroutine in C code",
+}
+
+// Identify whether the stack trace entry is part of ignoredStackFn .
+func isIgnoredStackFn(stack string) (ok bool) {
+	ok = true
+	for _, stackFn := range ignoredStackFns {
+		if !strings.Contains(stack, stackFn) {
+			ok = false
+			continue
+		}
+		break
+	}
+	return ok
+}
+
+// pickRelevantGoroutines returns all goroutines we care about for the purpose
+// of leak checking. It excludes testing or runtime ones.
+func pickRelevantGoroutines() (gs []string) {
+	// make a large buffer to hold the runtime stack info.
+	buf := make([]byte, 2<<20)
+	buf = buf[:runtime.Stack(buf, true)]
+	// runtime stack of go routines will be listed with 2 blank spaces between each of them, so split on "\n\n" .
+	for _, g := range strings.Split(string(buf), "\n\n") {
+		// Again split on a new line, the first line of the second half contaisn the info about the go routine.
+		sl := strings.SplitN(g, "\n", 2)
+		if len(sl) != 2 {
+			continue
+		}
+		stack := strings.TrimSpace(sl[1])
+		// ignore the testing go routine.
+		// since the tests will be invoking the leaktest it would contain the test go routine.
+		if strings.HasPrefix(stack, "testing.RunTests") {
+			continue
+		}
+		// Ignore the following go routines.
+		// testing and run time go routines should be ignored, only the application generated go routines should be taken into account.
+		if isIgnoredStackFn(stack) {
+			continue
+		}
+		gs = append(gs, g)
+	}
+	sort.Strings(gs)
+	return
+}

cmd/logger-console-hook.go

@@ -0,0 +1,46 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"io/ioutil"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// consoleLogger - default logger if not other logging is enabled.
+type consoleLogger struct {
+	Enable bool   `json:"enable"`
+	Level  string `json:"level"`
+}
+
+// enable console logger.
+func enableConsoleLogger() {
+	clogger := serverConfig.GetConsoleLogger()
+	if !clogger.Enable {
+		// Disable console logger if asked for.
+		log.Out = ioutil.Discard
+		return
+	}
+
+	// log.Out and log.Formatter use the default versions.
+	// Only set specific log level.
+	lvl, err := logrus.ParseLevel(clogger.Level)
+	fatalIf(err, "Unknown log level found in the config file.")
+
+	log.Level = lvl
+}

cmd/logger-file-hook.go

@@ -0,0 +1,75 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type fileLogger struct {
+	Enable   bool   `json:"enable"`
+	Filename string `json:"fileName"`
+	Level    string `json:"level"`
+}
+
+type localFile struct {
+	*os.File
+}
+
+func enableFileLogger() {
+	flogger := serverConfig.GetFileLogger()
+	if !flogger.Enable || flogger.Filename == "" {
+		return
+	}
+
+	// Creates the named file with mode 0666, honors system umask.
+	file, err := os.OpenFile(flogger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
+	fatalIf(err, "Unable to open log file.")
+
+	// Add a local file hook.
+	log.Hooks.Add(&localFile{file})
+
+	lvl, err := logrus.ParseLevel(flogger.Level)
+	fatalIf(err, "Unknown log level found in the config file.")
+
+	// Set default JSON formatter.
+	log.Formatter = new(logrus.JSONFormatter)
+	log.Level = lvl // Minimum log level.
+}
+
+// Fire fires the file logger hook and logs to the file.
+func (l *localFile) Fire(entry *logrus.Entry) error {
+	line, err := entry.String()
+	if err != nil {
+		return fmt.Errorf("Unable to read entry, %v", err)
+	}
+	l.File.Write([]byte(line + "\n"))
+	l.File.Sync()
+	return nil
+}
+
+// Levels - indicate log levels supported.
+func (l *localFile) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.PanicLevel,
+		logrus.FatalLevel,
+		logrus.ErrorLevel,
+	}
+}

cmd/logger-syslog-hook_nix.go

@@ -0,0 +1,82 @@
+// +build !windows
+
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"log/syslog"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type syslogLogger struct {
+	Enable bool   `json:"enable"`
+	Addr   string `json:"address"`
+	Level  string `json:"level"`
+}
+
+// syslogHook to send logs via syslog.
+type syslogHook struct {
+	writer        *syslog.Writer
+	syslogNetwork string
+	syslogRaddr   string
+}
+
+// enableSyslogLogger - enable logger at raddr.
+func enableSyslogLogger(raddr string) {
+	syslogHook, err := newSyslog("udp", raddr, syslog.LOG_ERR, "MINIO")
+	fatalIf(err, "Unable to initialize syslog logger.")
+
+	log.Hooks.Add(syslogHook)               // Add syslog hook.
+	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
+	log.Level = logrus.ErrorLevel           // Minimum log level.
+}
+
+// newSyslog - Creates a hook to be added to an instance of logger.
+func newSyslog(network, raddr string, priority syslog.Priority, tag string) (*syslogHook, error) {
+	w, err := syslog.Dial(network, raddr, priority, tag)
+	return &syslogHook{w, network, raddr}, err
+}
+
+// Fire - fire the log event
+func (hook *syslogHook) Fire(entry *logrus.Entry) error {
+	line, err := entry.String()
+	if err != nil {
+		return fmt.Errorf("Unable to read entry, %v", err)
+	}
+	switch entry.Level {
+	case logrus.PanicLevel:
+		return hook.writer.Crit(line)
+	case logrus.FatalLevel:
+		return hook.writer.Crit(line)
+	case logrus.ErrorLevel:
+		return hook.writer.Err(line)
+	default:
+		return nil
+	}
+}
+
+// Levels -
+func (hook *syslogHook) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.PanicLevel,
+		logrus.FatalLevel,
+		logrus.ErrorLevel,
+	}
+}

cmd/logger-syslog-hook_windows.go

@@ -0,0 +1,30 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+type syslogLogger struct {
+	Enable bool   `json:"enable"`
+	Addr   string `json:"address"`
+	Level  string `json:"level"`
+}
+
+// enableSyslogLogger - unsupported on windows.
+func enableSyslogLogger(raddr string) {
+	fatalIf(errSyslogNotSupported, "Unable to enable syslog.")
+}

cmd/logger.go

@@ -0,0 +1,88 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"runtime/debug"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+)
+
+type fields map[string]interface{}
+
+var log = logrus.New() // Default console logger.
+
+// logger carries logging configuration for various supported loggers.
+// Currently supported loggers are
+//
+//   - console [default]
+//   - file
+//   - syslog
+type logger struct {
+	Console consoleLogger `json:"console"`
+	File    fileLogger    `json:"file"`
+	Syslog  syslogLogger  `json:"syslog"`
+	// Add new loggers here.
+}
+
+// stackInfo returns printable stack trace.
+func stackInfo() string {
+	// Convert stack-trace bytes to io.Reader.
+	rawStack := bufio.NewReader(bytes.NewBuffer(debug.Stack()))
+	// Skip stack trace lines until our real caller.
+	for i := 0; i <= 4; i++ {
+		rawStack.ReadLine()
+	}
+
+	// Read the rest of useful stack trace.
+	stackBuf := new(bytes.Buffer)
+	stackBuf.ReadFrom(rawStack)
+
+	// Strip GOPATH of the build system and return.
+	return strings.Replace(stackBuf.String(), GOPATH+"/src/", "", -1)
+}
+
+// errorIf synonymous with fatalIf but doesn't exit on error != nil
+func errorIf(err error, msg string, data ...interface{}) {
+	if err == nil {
+		return
+	}
+	fields := logrus.Fields{
+		"cause": err.Error(),
+	}
+	if globalTrace {
+		fields["stack"] = "\n" + stackInfo()
+	}
+	log.WithFields(fields).Errorf(msg, data...)
+}
+
+// fatalIf wrapper function which takes error and prints jsonic error messages.
+func fatalIf(err error, msg string, data ...interface{}) {
+	if err == nil {
+		return
+	}
+	fields := logrus.Fields{
+		"cause": err.Error(),
+	}
+	if globalTrace {
+		fields["stack"] = "\n" + stackInfo()
+	}
+	log.WithFields(fields).Fatalf(msg, data...)
+}

cmd/logger_test.go

@@ -0,0 +1,47 @@
+/*
+ * Minio Cloud Storage (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+
+	"github.com/Sirupsen/logrus"
+
+	. "gopkg.in/check.v1"
+)
+
+type LoggerSuite struct{}
+
+var _ = Suite(&LoggerSuite{})
+
+func (s *LoggerSuite) TestLogger(c *C) {
+	var buffer bytes.Buffer
+	var fields logrus.Fields
+	log.Out = &buffer
+	log.Formatter = new(logrus.JSONFormatter)
+
+	errorIf(errors.New("Fake error"), "Failed with error.")
+	err := json.Unmarshal(buffer.Bytes(), &fields)
+	c.Assert(err, IsNil)
+	c.Assert(fields["level"], Equals, "error")
+
+	msg, ok := fields["cause"]
+	c.Assert(ok, Equals, true)
+	c.Assert(msg, Equals, "Fake error")
+}

cmd/main.go

@@ -0,0 +1,193 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"sort"
+
+	"github.com/minio/cli"
+	"github.com/minio/mc/pkg/console"
+	"github.com/pkg/profile"
+)
+
+var (
+	// global flags for minio.
+	minioFlags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "help, h",
+			Usage: "Show help.",
+		},
+	}
+)
+
+// Help template for minio.
+var minioHelpTemplate = `NAME:
+  {{.Name}} - {{.Usage}}
+
+DESCRIPTION:
+  {{.Description}}
+
+USAGE:
+  minio {{if .Flags}}[flags] {{end}}command{{if .Flags}}{{end}} [arguments...]
+
+COMMANDS:
+  {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
+  {{end}}{{if .Flags}}
+FLAGS:
+  {{range .Flags}}{{.}}
+  {{end}}{{end}}
+VERSION:
+  ` + Version +
+	`{{ "\n"}}`
+
+// init - check the environment before main starts
+func init() {
+	// Check if minio was compiled using a supported version of Golang.
+	checkGoVersion()
+
+	// Set global trace flag.
+	globalTrace = os.Getenv("MINIO_TRACE") == "1"
+}
+
+func migrate() {
+	// Migrate config file
+	migrateConfig()
+
+	// Migrate other configs here.
+}
+
+func enableLoggers() {
+	// Enable all loggers here.
+	enableConsoleLogger()
+	enableFileLogger()
+	// Add your logger here.
+}
+
+func findClosestCommands(command string) []string {
+	var closestCommands []string
+	for _, value := range commandsTree.PrefixMatch(command) {
+		closestCommands = append(closestCommands, value.(string))
+	}
+	sort.Strings(closestCommands)
+	// Suggest other close commands - allow missed, wrongly added and
+	// even transposed characters
+	for _, value := range commandsTree.walk(commandsTree.root) {
+		if sort.SearchStrings(closestCommands, value.(string)) < len(closestCommands) {
+			continue
+		}
+		// 2 is arbitrary and represents the max
+		// allowed number of typed errors
+		if DamerauLevenshteinDistance(command, value.(string)) < 2 {
+			closestCommands = append(closestCommands, value.(string))
+		}
+	}
+	return closestCommands
+}
+
+func registerApp() *cli.App {
+	// Register all commands.
+	registerCommand(serverCmd)
+	registerCommand(versionCmd)
+	registerCommand(updateCmd)
+	registerCommand(controlCmd)
+
+	// Set up app.
+	app := cli.NewApp()
+	app.Name = "Minio"
+	app.Author = "Minio.io"
+	app.Usage = "Cloud Storage Server."
+	app.Description = `Minio is an Amazon S3 compatible object storage server. Use it to store photos, videos, VMs, containers, log files, or any blob of data as objects.`
+	app.Flags = append(minioFlags, globalFlags...)
+	app.Commands = commands
+	app.CustomAppHelpTemplate = minioHelpTemplate
+	app.CommandNotFound = func(ctx *cli.Context, command string) {
+		msg := fmt.Sprintf("‘%s’ is not a minio sub-command. See ‘minio --help’.", command)
+		closestCommands := findClosestCommands(command)
+		if len(closestCommands) > 0 {
+			msg += fmt.Sprintf("\n\nDid you mean one of these?\n")
+			for _, cmd := range closestCommands {
+				msg += fmt.Sprintf("        ‘%s’\n", cmd)
+			}
+		}
+		console.Fatalln(msg)
+	}
+	return app
+}
+
+func checkMainSyntax(c *cli.Context) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		console.Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
+	}
+	if configPath == "" {
+		console.Fatalln("Config folder cannot be empty, please specify --config-dir <foldername>.")
+	}
+}
+
+// Main - main for minio server.
+func Main() {
+	app := registerApp()
+	app.Before = func(c *cli.Context) error {
+		// Sets new config folder.
+		setGlobalConfigPath(c.GlobalString("config-dir"))
+
+		// Valid input arguments to main.
+		checkMainSyntax(c)
+
+		// Migrate any old version of config / state files to newer format.
+		migrate()
+
+		// Initialize config.
+		err := initConfig()
+		fatalIf(err, "Unable to initialize minio config.")
+
+		// Enable all loggers by now.
+		enableLoggers()
+
+		// Set global quiet flag.
+		globalQuiet = c.Bool("quiet") || c.GlobalBool("quiet")
+
+		// Do not print update messages, if quiet flag is set.
+		if !globalQuiet {
+			// Do not print any errors in release update function.
+			noError := true
+			updateMsg := getReleaseUpdate(minioUpdateStableURL, noError)
+			if updateMsg.Update {
+				console.Println(updateMsg)
+			}
+		}
+
+		return nil
+	}
+
+	// Set ``MINIO_PROFILE_DIR`` to the directory where profiling information should be persisted
+	profileDir := os.Getenv("MINIO_PROFILE_DIR")
+	// Enable profiler if ``MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
+	switch os.Getenv("MINIO_PROFILER") {
+	case "cpu":
+		defer profile.Start(profile.CPUProfile, profile.ProfilePath(profileDir)).Stop()
+	case "mem":
+		defer profile.Start(profile.MemProfile, profile.ProfilePath(profileDir)).Stop()
+	case "block":
+		defer profile.Start(profile.BlockProfile, profile.ProfilePath(profileDir)).Stop()
+	}
+
+	// Run the app - exit on error.
+	app.RunAndExitOnError()
+}

cmd/main_test.go

@@ -0,0 +1,26 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }

cmd/namespace-lock.go

@@ -0,0 +1,126 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"sync"
+)
+
+// nsParam - carries name space resource.
+type nsParam struct {
+	volume string
+	path   string
+}
+
+// nsLock - provides primitives for locking critical namespace regions.
+type nsLock struct {
+	sync.RWMutex
+	ref uint
+}
+
+// nsLockMap - namespace lock map, provides primitives to Lock,
+// Unlock, RLock and RUnlock.
+type nsLockMap struct {
+	lockMap map[nsParam]*nsLock
+	mutex   sync.Mutex
+}
+
+// Global name space lock.
+var nsMutex *nsLockMap
+
+// initNSLock - initialize name space lock map.
+func initNSLock() {
+	nsMutex = &nsLockMap{
+		lockMap: make(map[nsParam]*nsLock),
+	}
+}
+
+// Lock the namespace resource.
+func (n *nsLockMap) lock(volume, path string, readLock bool) {
+	n.mutex.Lock()
+
+	param := nsParam{volume, path}
+	nsLk, found := n.lockMap[param]
+	if !found {
+		nsLk = &nsLock{
+			ref: 0,
+		}
+		n.lockMap[param] = nsLk
+	}
+	nsLk.ref++ // Update ref count here to avoid multiple races.
+	// Unlock map before Locking NS which might block.
+	n.mutex.Unlock()
+
+	// Locking here can block.
+	if readLock {
+		nsLk.RLock()
+	} else {
+		nsLk.Lock()
+	}
+}
+
+// Unlock the namespace resource.
+func (n *nsLockMap) unlock(volume, path string, readLock bool) {
+	// nsLk.Unlock() will not block, hence locking the map for the entire function is fine.
+	n.mutex.Lock()
+	defer n.mutex.Unlock()
+
+	param := nsParam{volume, path}
+	if nsLk, found := n.lockMap[param]; found {
+		if readLock {
+			nsLk.RUnlock()
+		} else {
+			nsLk.Unlock()
+		}
+		if nsLk.ref == 0 {
+			errorIf(errors.New("Namespace reference count cannot be 0."), "Invalid reference count detected.")
+		}
+		if nsLk.ref != 0 {
+			nsLk.ref--
+		}
+		if nsLk.ref == 0 {
+			// Remove from the map if there are no more references.
+			delete(n.lockMap, param)
+		}
+	}
+}
+
+// Lock - locks the given resource for writes, using a previously
+// allocated name space lock or initializing a new one.
+func (n *nsLockMap) Lock(volume, path string) {
+	readLock := false
+	n.lock(volume, path, readLock)
+}
+
+// Unlock - unlocks any previously acquired write locks.
+func (n *nsLockMap) Unlock(volume, path string) {
+	readLock := false
+	n.unlock(volume, path, readLock)
+}
+
+// RLock - locks any previously acquired read locks.
+func (n *nsLockMap) RLock(volume, path string) {
+	readLock := true
+	n.lock(volume, path, readLock)
+}
+
+// RUnlock - unlocks any previously acquired read locks.
+func (n *nsLockMap) RUnlock(volume, path string) {
+	readLock := true
+	n.unlock(volume, path, readLock)
+}

cmd/namespace-lock_test.go

@@ -0,0 +1,123 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "testing"
+
+// Tests functionality provided by namespace lock.
+func TestNamespaceLockTest(t *testing.T) {
+	// List of test cases.
+	testCases := []struct {
+		lk               func(s1, s2 string)
+		unlk             func(s1, s2 string)
+		rlk              func(s1, s2 string)
+		runlk            func(s1, s2 string)
+		lkCount          int
+		lockedRefCount   uint
+		unlockedRefCount uint
+		shouldPass       bool
+	}{
+		{
+			lk:               nsMutex.Lock,
+			unlk:             nsMutex.Unlock,
+			lockedRefCount:   1,
+			unlockedRefCount: 0,
+			shouldPass:       true,
+		},
+		{
+			rlk:              nsMutex.RLock,
+			runlk:            nsMutex.RUnlock,
+			lockedRefCount:   4,
+			unlockedRefCount: 2,
+			shouldPass:       true,
+		},
+		{
+			rlk:              nsMutex.RLock,
+			runlk:            nsMutex.RUnlock,
+			lockedRefCount:   1,
+			unlockedRefCount: 0,
+			shouldPass:       true,
+		},
+	}
+
+	// Run all test cases.
+
+	// Write lock tests.
+	testCase := testCases[0]
+	testCase.lk("a", "b") // lock once.
+	nsLk, ok := nsMutex.lockMap[nsParam{"a", "b"}]
+	if !ok && testCase.shouldPass {
+		t.Errorf("Lock in map missing.")
+	}
+	// Validate loced ref count.
+	if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.lockedRefCount, nsLk.ref)
+	}
+	testCase.unlk("a", "b") // unlock once.
+	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.unlockedRefCount, nsLk.ref)
+	}
+	_, ok = nsMutex.lockMap[nsParam{"a", "b"}]
+	if ok && !testCase.shouldPass {
+		t.Errorf("Lock map found after unlock.")
+	}
+
+	// Read lock tests.
+	testCase = testCases[1]
+	testCase.rlk("a", "b") // lock once.
+	testCase.rlk("a", "b") // lock second time.
+	testCase.rlk("a", "b") // lock third time.
+	testCase.rlk("a", "b") // lock fourth time.
+	nsLk, ok = nsMutex.lockMap[nsParam{"a", "b"}]
+	if !ok && testCase.shouldPass {
+		t.Errorf("Lock in map missing.")
+	}
+	// Validate loced ref count.
+	if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.lockedRefCount, nsLk.ref)
+	}
+	testCase.runlk("a", "b") // unlock once.
+	testCase.runlk("a", "b") // unlock second time.
+	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 2, testCase.unlockedRefCount, nsLk.ref)
+	}
+	_, ok = nsMutex.lockMap[nsParam{"a", "b"}]
+	if !ok && testCase.shouldPass {
+		t.Errorf("Lock map not found.")
+	}
+
+	// Read lock 0 ref count.
+	testCase = testCases[2]
+	testCase.rlk("a", "c") // lock once.
+
+	nsLk, ok = nsMutex.lockMap[nsParam{"a", "c"}]
+	if !ok && testCase.shouldPass {
+		t.Errorf("Lock in map missing.")
+	}
+	// Validate loced ref count.
+	if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 3, testCase.lockedRefCount, nsLk.ref)
+	}
+	testCase.runlk("a", "c") // unlock once.
+	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
+		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 3, testCase.unlockedRefCount, nsLk.ref)
+	}
+	_, ok = nsMutex.lockMap[nsParam{"a", "c"}]
+	if ok && !testCase.shouldPass {
+		t.Errorf("Lock map not found.")
+	}
+}

cmd/notifiers.go

@@ -0,0 +1,136 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/minio/minio/pkg/wildcard"
+)
+
+// SQS type.
+const (
+	// Minio sqs ARN prefix.
+	minioSqs = "arn:minio:sqs:"
+
+	// Static string indicating queue type 'amqp'.
+	queueTypeAMQP = "amqp"
+	// Static string indicating queue type 'elasticsearch'.
+	queueTypeElastic = "elasticsearch"
+	// Static string indicating queue type 'redis'.
+	queueTypeRedis = "redis"
+)
+
+// Topic type.
+const (
+	// Minio topic ARN prefix.
+	minioTopic = "arn:minio:sns:"
+
+	// Static string indicating sns type 'listen'.
+	snsTypeMinio = "listen"
+)
+
+var errNotifyNotEnabled = errors.New("requested notifier not enabled")
+
+// Notifier represents collection of supported notification queues.
+type notifier struct {
+	AMQP          map[string]amqpNotify          `json:"amqp"`
+	ElasticSearch map[string]elasticSearchNotify `json:"elasticsearch"`
+	Redis         map[string]redisNotify         `json:"redis"`
+	// Add new notification queues.
+}
+
+// Returns true if queueArn is for an AMQP queue.
+func isAMQPQueue(sqsArn arnSQS) bool {
+	if sqsArn.Type != queueTypeAMQP {
+		return false
+	}
+	amqpL := serverConfig.GetAMQPNotifyByID(sqsArn.AccountID)
+	if !amqpL.Enable {
+		return false
+	}
+	// Connect to amqp server to validate.
+	amqpC, err := dialAMQP(amqpL)
+	if err != nil {
+		errorIf(err, "Unable to connect to amqp service. %#v", amqpL)
+		return false
+	}
+	defer amqpC.Close()
+	return true
+}
+
+// Returns true if queueArn is for an Redis queue.
+func isRedisQueue(sqsArn arnSQS) bool {
+	if sqsArn.Type != queueTypeRedis {
+		return false
+	}
+	rNotify := serverConfig.GetRedisNotifyByID(sqsArn.AccountID)
+	if !rNotify.Enable {
+		return false
+	}
+	// Connect to redis server to validate.
+	rPool, err := dialRedis(rNotify)
+	if err != nil {
+		errorIf(err, "Unable to connect to redis service. %#v", rNotify)
+		return false
+	}
+	defer rPool.Close()
+	return true
+}
+
+// Returns true if queueArn is for an ElasticSearch queue.
+func isElasticQueue(sqsArn arnSQS) bool {
+	if sqsArn.Type != queueTypeElastic {
+		return false
+	}
+	esNotify := serverConfig.GetElasticSearchNotifyByID(sqsArn.AccountID)
+	if !esNotify.Enable {
+		return false
+	}
+	elasticC, err := dialElastic(esNotify)
+	if err != nil {
+		errorIf(err, "Unable to connect to elasticsearch service %#v", esNotify)
+		return false
+	}
+	defer elasticC.Stop()
+	return true
+}
+
+// Match function matches wild cards in 'pattern' for events.
+func eventMatch(eventType string, events []string) (ok bool) {
+	for _, event := range events {
+		ok = wildcard.Match(event, eventType)
+		if ok {
+			break
+		}
+	}
+	return ok
+}
+
+// Filter rule match, matches an object against the filter rules.
+func filterRuleMatch(object string, frs []filterRule) bool {
+	var prefixMatch, suffixMatch = true, true
+	for _, fr := range frs {
+		if isValidFilterNamePrefix(fr.Name) {
+			prefixMatch = strings.HasPrefix(object, fr.Value)
+		} else if isValidFilterNameSuffix(fr.Name) {
+			suffixMatch = strings.HasSuffix(object, fr.Value)
+		}
+	}
+	return prefixMatch && suffixMatch
+}

cmd/notifiers_test.go

@@ -0,0 +1,107 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "testing"
+
+// Tests all event match.
+func TestEventMatch(t *testing.T) {
+	testCases := []struct {
+		eventName EventName
+		events    []string
+		match     bool
+	}{
+		// Valid object created PUT event.
+		{
+			eventName: ObjectCreatedPut,
+			events: []string{
+				"s3:ObjectCreated:Put",
+			},
+			match: true,
+		},
+		// Valid object removed DELETE event.
+		{
+			eventName: ObjectRemovedDelete,
+			events: []string{
+				"s3:ObjectRemoved:Delete",
+			},
+			match: true,
+		},
+		// Invalid events fails to match with empty events.
+		{
+			eventName: ObjectRemovedDelete,
+			events:    []string{""},
+			match:     false,
+		},
+		// Invalid events fails to match with valid events.
+		{
+			eventName: ObjectCreatedCompleteMultipartUpload,
+			events: []string{
+				"s3:ObjectRemoved:*",
+			},
+			match: false,
+		},
+		// Valid events wild card match.
+		{
+			eventName: ObjectCreatedPut,
+			events: []string{
+				"s3:ObjectCreated:*",
+			},
+			match: true,
+		},
+		// Valid events wild card match.
+		{
+			eventName: ObjectCreatedPost,
+			events: []string{
+				"s3:ObjectCreated:*",
+			},
+			match: true,
+		},
+		// Valid events wild card match.
+		{
+			eventName: ObjectCreatedCopy,
+			events: []string{
+				"s3:ObjectCreated:*",
+			},
+			match: true,
+		},
+		// Valid events wild card match.
+		{
+			eventName: ObjectCreatedCompleteMultipartUpload,
+			events: []string{
+				"s3:ObjectCreated:*",
+			},
+			match: true,
+		},
+		// Valid events wild card match.
+		{
+			eventName: ObjectCreatedPut,
+			events: []string{
+				"s3:ObjectCreated:*",
+				"s3:ObjectRemoved:*",
+			},
+			match: true,
+		},
+	}
+
+	for i, testCase := range testCases {
+		ok := eventMatch(testCase.eventName.String(), testCase.events)
+		if testCase.match != ok {
+			t.Errorf("Test %d: Expected \"%t\", got \"%t\"", i+1, testCase.match, ok)
+		}
+	}
+}

cmd/notify-amqp.go

@@ -0,0 +1,145 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"io/ioutil"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/streadway/amqp"
+)
+
+// amqpNotify - represents logrus compatible AMQP hook.
+// All fields represent AMQP configuration details.
+type amqpNotify struct {
+	Enable       bool   `json:"enable"`
+	URL          string `json:"url"`
+	Exchange     string `json:"exchange"`
+	RoutingKey   string `json:"routingKey"`
+	ExchangeType string `json:"exchangeType"`
+	Mandatory    bool   `json:"mandatory"`
+	Immediate    bool   `json:"immediate"`
+	Durable      bool   `json:"durable"`
+	Internal     bool   `json:"internal"`
+	NoWait       bool   `json:"noWait"`
+	AutoDeleted  bool   `json:"autoDeleted"`
+}
+
+type amqpConn struct {
+	params amqpNotify
+	*amqp.Connection
+}
+
+// dialAMQP - dials and returns an amqpConnection instance,
+// for sending notifications. Returns error if amqp logger
+// is not enabled.
+func dialAMQP(amqpL amqpNotify) (amqpConn, error) {
+	if !amqpL.Enable {
+		return amqpConn{}, errNotifyNotEnabled
+	}
+	conn, err := amqp.Dial(amqpL.URL)
+	if err != nil {
+		return amqpConn{}, err
+	}
+	return amqpConn{Connection: conn, params: amqpL}, nil
+}
+
+func newAMQPNotify(accountID string) (*logrus.Logger, error) {
+	amqpL := serverConfig.GetAMQPNotifyByID(accountID)
+
+	// Connect to amqp server.
+	amqpC, err := dialAMQP(amqpL)
+	if err != nil {
+		return nil, err
+	}
+
+	amqpLog := logrus.New()
+
+	// Disable writing to console.
+	amqpLog.Out = ioutil.Discard
+
+	// Add a amqp hook.
+	amqpLog.Hooks.Add(amqpC)
+
+	// Set default JSON formatter.
+	amqpLog.Formatter = new(logrus.JSONFormatter)
+
+	// Successfully enabled all AMQPs.
+	return amqpLog, nil
+}
+
+// Fire is called when an event should be sent to the message broker.k
+func (q amqpConn) Fire(entry *logrus.Entry) error {
+	ch, err := q.Connection.Channel()
+	if err != nil {
+		// Any other error other than connection closed, return.
+		if err != amqp.ErrClosed {
+			return err
+		}
+		// Attempt to connect again.
+		var conn *amqp.Connection
+		conn, err = amqp.Dial(q.params.URL)
+		if err != nil {
+			return err
+		}
+		ch, err = conn.Channel()
+		if err != nil {
+			return err
+		}
+	}
+	defer ch.Close()
+
+	err = ch.ExchangeDeclare(
+		q.params.Exchange,
+		q.params.ExchangeType,
+		q.params.Durable,
+		q.params.AutoDeleted,
+		q.params.Internal,
+		q.params.NoWait,
+		nil,
+	)
+	if err != nil {
+		return err
+	}
+
+	body, err := entry.String()
+	if err != nil {
+		return err
+	}
+
+	err = ch.Publish(
+		q.params.Exchange,
+		q.params.RoutingKey,
+		q.params.Mandatory,
+		q.params.Immediate,
+		amqp.Publishing{
+			ContentType: "application/json",
+			Body:        []byte(body),
+		})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Levels is available logging levels.
+func (q amqpConn) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.InfoLevel,
+	}
+}

cmd/notify-elasticsearch.go

@@ -0,0 +1,112 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"io/ioutil"
+
+	"github.com/Sirupsen/logrus"
+	"gopkg.in/olivere/elastic.v3"
+)
+
+// elasticQueue is a elasticsearch event notification queue.
+type elasticSearchNotify struct {
+	Enable bool   `json:"enable"`
+	URL    string `json:"url"`
+	Index  string `json:"index"`
+}
+
+type elasticClient struct {
+	*elastic.Client
+	params elasticSearchNotify
+}
+
+// Connects to elastic search instance at URL.
+func dialElastic(esNotify elasticSearchNotify) (*elastic.Client, error) {
+	if !esNotify.Enable {
+		return nil, errNotifyNotEnabled
+	}
+	client, err := elastic.NewClient(elastic.SetURL(esNotify.URL), elastic.SetSniff(false))
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+func newElasticNotify(accountID string) (*logrus.Logger, error) {
+	esNotify := serverConfig.GetElasticSearchNotifyByID(accountID)
+
+	// Dial to elastic search.
+	client, err := dialElastic(esNotify)
+	if err != nil {
+		return nil, err
+	}
+
+	// Use the IndexExists service to check if a specified index exists.
+	exists, err := client.IndexExists(esNotify.Index).Do()
+	if err != nil {
+		return nil, err
+	}
+	// Index does not exist, attempt to create it.
+	if !exists {
+		var createIndex *elastic.IndicesCreateResult
+		createIndex, err = client.CreateIndex(esNotify.Index).Do()
+		if err != nil {
+			return nil, err
+		}
+		if !createIndex.Acknowledged {
+			return nil, errors.New("index not created")
+		}
+	}
+
+	elasticCl := elasticClient{
+		Client: client,
+		params: esNotify,
+	}
+
+	elasticSearchLog := logrus.New()
+
+	// Disable writing to console.
+	elasticSearchLog.Out = ioutil.Discard
+
+	// Add a elasticSearch hook.
+	elasticSearchLog.Hooks.Add(elasticCl)
+
+	// Set default JSON formatter.
+	elasticSearchLog.Formatter = new(logrus.JSONFormatter)
+
+	// Success, elastic search successfully initialized.
+	return elasticSearchLog, nil
+}
+
+// Fire is required to implement logrus hook
+func (q elasticClient) Fire(entry *logrus.Entry) error {
+	_, err := q.Client.Index().Index(q.params.Index).
+		Type("event").
+		BodyJson(entry.Data).
+		Do()
+
+	return err
+}
+
+// Required for logrus hook implementation
+func (q elasticClient) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.InfoLevel,
+	}
+}

cmd/notify-redis.go

@@ -0,0 +1,133 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"io/ioutil"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/redigo/redis"
+)
+
+// redisNotify to send logs to Redis server
+type redisNotify struct {
+	Enable   bool   `json:"enable"`
+	Addr     string `json:"address"`
+	Password string `json:"password"`
+	Key      string `json:"key"`
+}
+
+type redisConn struct {
+	*redis.Pool
+	params redisNotify
+}
+
+// Dial a new connection to redis instance at addr, optionally with a password if any.
+func dialRedis(rNotify redisNotify) (*redis.Pool, error) {
+	// Return error if redis not enabled.
+	if !rNotify.Enable {
+		return nil, errNotifyNotEnabled
+	}
+	addr := rNotify.Addr
+	password := rNotify.Password
+	rPool := &redis.Pool{
+		MaxIdle:     3,
+		IdleTimeout: 240 * time.Second,
+		Dial: func() (redis.Conn, error) {
+			c, err := redis.Dial("tcp", addr)
+			if err != nil {
+				return nil, err
+			}
+			if password != "" {
+				if _, derr := c.Do("AUTH", password); derr != nil {
+					c.Close()
+					return nil, derr
+				}
+			}
+			return c, err
+		},
+		TestOnBorrow: func(c redis.Conn, t time.Time) error {
+			_, err := c.Do("PING")
+			return err
+		},
+	}
+
+	// Test if connection with REDIS can be established.
+	rConn := rPool.Get()
+	defer rConn.Close()
+
+	// Check connection.
+	_, err := rConn.Do("PING")
+	if err != nil {
+		return nil, err
+	}
+
+	// Return pool.
+	return rPool, nil
+}
+
+func newRedisNotify(accountID string) (*logrus.Logger, error) {
+	rNotify := serverConfig.GetRedisNotifyByID(accountID)
+
+	// Dial redis.
+	rPool, err := dialRedis(rNotify)
+	if err != nil {
+		return nil, err
+	}
+
+	rrConn := redisConn{
+		Pool:   rPool,
+		params: rNotify,
+	}
+
+	redisLog := logrus.New()
+
+	redisLog.Out = ioutil.Discard
+
+	// Set default JSON formatter.
+	redisLog.Formatter = new(logrus.JSONFormatter)
+
+	redisLog.Hooks.Add(rrConn)
+
+	// Success, redis enabled.
+	return redisLog, nil
+}
+
+// Fire is called when an event should be sent to the message broker.
+func (r redisConn) Fire(entry *logrus.Entry) error {
+	rConn := r.Pool.Get()
+	defer rConn.Close()
+
+	data, err := entry.String()
+	if err != nil {
+		return err
+	}
+
+	_, err = rConn.Do("RPUSH", r.params.Key, data)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Required for logrus hook implementation
+func (r redisConn) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.InfoLevel,
+	}
+}

cmd/object-api-getobject_test.go

@@ -0,0 +1,500 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+)
+
+// Wrapper for calling GetObject tests for both XL multiple disks and single node setup.
+func TestGetObject(t *testing.T) {
+	ExecObjectLayerTest(t, testGetObject)
+}
+
+// ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated.
+func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// Setup for the tests.
+	bucketName := getRandomBucketName()
+	objectName := "test-object"
+	// create bucket.
+	err := obj.MakeBucket(bucketName)
+	// Stop the test if creation of the bucket fails.
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// set of byte data for PutObject.
+	// object has to be inserted before running tests for GetObject.
+	// this is required even to assert the GetObject data,
+	// since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical.
+	bytesData := []struct {
+		byteData []byte
+	}{
+		{generateBytesData(6 * 1024 * 1024)},
+	}
+	// set of inputs for uploading the objects before tests for downloading is done.
+	putObjectInputs := []struct {
+		bucketName    string
+		objectName    string
+		contentLength int64
+		textData      []byte
+		metaData      map[string]string
+	}{
+		// case - 1.
+		{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
+	}
+	// iterate through the above set of inputs and upkoad the object.
+	for i, input := range putObjectInputs {
+		// uploading the object.
+		_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
+		// if object upload fails stop the test.
+		if err != nil {
+			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
+		}
+	}
+	// set of empty buffers used to fill GetObject data.
+	buffers := []*bytes.Buffer{
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+	}
+
+	// test cases with set of inputs
+	testCases := []struct {
+		bucketName  string
+		objectName  string
+		startOffset int64
+		length      int64
+		// data obtained/fetched from GetObject.
+		getObjectData *bytes.Buffer
+		// writer which governs the write into the `getObjectData`.
+		writer io.Writer
+		// flag indicating whether the test for given ase should pass.
+		shouldPass bool
+		// expected Result.
+		expectedData []byte
+		err          error
+	}{
+		// Test case 1-4.
+		// Cases with invalid bucket names.
+		{".test", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: .test")},
+		{"------", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: ------")},
+		{"$this-is-not-valid-too", "obj", 0, 0, nil, nil, false,
+			[]byte(""), fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")},
+		{"a", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: a")},
+		// Test case - 5.
+		// Case with invalid object names.
+		{bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"#")},
+		// Test case - 6.
+		// 	Valid object and bucket names but non-existent bucket.
+		//	{"abc", "def", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket not found: abc")},
+		// A custom writer is sent as an argument.
+		// Its designed to return a EOF error after reading `n` bytes, where `n` is the argument when initializing the EOF writer.
+		// This is to simulate the case of cache not filling up completly, since the EOFWriter doesn't allow the write to complete,
+		// the cache gets filled up with partial data. The following up test case will read the object completly, tests the
+		// purging of the cache during the incomplete write.
+		//	Test case - 7.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[0], NewEOFWriter(buffers[0], 100), false, []byte{}, io.EOF},
+		// Test case with start offset set to 0 and length set to size of the object.
+		// Fetching the entire object.
+		// 	Test case - 8.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], true, bytesData[0].byteData, nil},
+		// Test case with content-range 1 to objectSize .
+		// Test case - 9.
+		{bucketName, objectName, 1, int64(len(bytesData[0].byteData) - 1), buffers[1], buffers[1], true, bytesData[0].byteData[1:], nil},
+		// Test case with content-range 100 to objectSize - 100.
+		// Test case - 10.
+		{bucketName, objectName, 100, int64(len(bytesData[0].byteData) - 200), buffers[1], buffers[1], true,
+			bytesData[0].byteData[100 : len(bytesData[0].byteData)-100], nil},
+		// Test case with offset greater than the size of the object
+		// Test case - 11.
+		{bucketName, objectName, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), buffers[0],
+			NewEOFWriter(buffers[0], 100), false, []byte{},
+			InvalidRange{int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}},
+		// Test case with offset greater than the size of the object.
+		// Test case - 12.
+		{bucketName, objectName, -1, int64(len(bytesData[0].byteData)), buffers[0], new(bytes.Buffer), false, []byte{}, errUnexpected},
+		// Test case length parameter is more than the object size.
+		// Test case - 13.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData) + 1), buffers[1], buffers[1], false, bytesData[0].byteData,
+			InvalidRange{0, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData))}},
+		// Test case with `length` parameter set to a negative value.
+		// Test case - 14.
+		{bucketName, objectName, 0, int64(-1), buffers[1], buffers[1], false, bytesData[0].byteData, errUnexpected},
+		// Test case with offset + length > objectSize parameter set to a negative value.
+		// Test case - 15.
+		{bucketName, objectName, 2, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], false, bytesData[0].byteData,
+			InvalidRange{2, int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}},
+		// Test case with the writer set to nil.
+		// Test case - 16.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], nil, false, bytesData[0].byteData, errUnexpected},
+	}
+
+	for i, testCase := range testCases {
+		err = obj.GetObject(testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead.", i+1, instanceType, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if !strings.Contains(err.Error(), testCase.err.Error()) {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.err.Error(), err.Error())
+			}
+		}
+		// Since there are cases for which GetObject fails, this is
+		// necessary. Test passes as expected, but the output values
+		// are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if !bytes.Equal(testCase.expectedData, testCase.getObjectData.Bytes()) {
+				t.Errorf("Test %d: %s: Data Mismatch: Expected data and the fetched data from GetObject doesn't match.", i+1, instanceType)
+			}
+			// empty the buffer so that it can be used to further cases.
+			testCase.getObjectData.Reset()
+		}
+	}
+}
+
+// Wrapper for calling GetObject tests for both XL multiple disks and single node setup.
+func TestGetObjectDiskNotFound(t *testing.T) {
+	ExecObjectLayerDiskNotFoundTest(t, testGetObjectDiskNotFound)
+}
+
+// ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated.
+// Before the Get Object call XL disks are moved so that the quorum just holds.
+func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
+	// Setup for the tests.
+	bucketName := getRandomBucketName()
+	objectName := "test-object"
+	// create bucket.
+	err := obj.MakeBucket(bucketName)
+	// Stop the test if creation of the bucket fails.
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// set of byte data for PutObject.
+	// object has to be inserted before running tests for GetObject.
+	// this is required even to assert the GetObject data,
+	// since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical.
+	bytesData := []struct {
+		byteData []byte
+	}{
+		{generateBytesData(6 * 1024 * 1024)},
+	}
+	// set of inputs for uploading the objects before tests for downloading is done.
+	putObjectInputs := []struct {
+		bucketName    string
+		objectName    string
+		contentLength int64
+		textData      []byte
+		metaData      map[string]string
+	}{
+		// case - 1.
+		{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
+	}
+	// iterate through the above set of inputs and upkoad the object.
+	for i, input := range putObjectInputs {
+		// uploading the object.
+		_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
+		// if object upload fails stop the test.
+		if err != nil {
+			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
+		}
+	}
+
+	// Take 8 disks down before GetObject is called, one more we loose quorum on 16 disk node.
+	for _, disk := range disks[:8] {
+		removeAll(disk)
+	}
+
+	// set of empty buffers used to fill GetObject data.
+	buffers := []*bytes.Buffer{
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+	}
+
+	// test cases with set of inputs
+	testCases := []struct {
+		bucketName  string
+		objectName  string
+		startOffset int64
+		length      int64
+		// data obtained/fetched from GetObject.
+		getObjectData *bytes.Buffer
+		// writer which governs the write into the `getObjectData`.
+		writer io.Writer
+		// flag indicating whether the test for given ase should pass.
+		shouldPass bool
+		// expected Result.
+		expectedData []byte
+		err          error
+	}{
+		// Test case 1-4.
+		// Cases with invalid bucket names.
+		{".test", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: .test")},
+		{"------", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: ------")},
+		{"$this-is-not-valid-too", "obj", 0, 0, nil, nil, false,
+			[]byte(""), fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")},
+		{"a", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: a")},
+		// Test case - 5.
+		// Case with invalid object names.
+		{bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"#")},
+		// Test case - 6.
+		// 	Valid object and bucket names but non-existent bucket.
+		//	{"abc", "def", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket not found: abc")},
+		// A custom writer is sent as an argument.
+		// Its designed to return a EOF error after reading `n` bytes, where `n` is the argument when initializing the EOF writer.
+		// This is to simulate the case of cache not filling up completly, since the EOFWriter doesn't allow the write to complete,
+		// the cache gets filled up with partial data. The following up test case will read the object completly, tests the
+		// purging of the cache during the incomplete write.
+		//	Test case - 7.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[0], NewEOFWriter(buffers[0], 100), false, []byte{}, io.EOF},
+		// Test case with start offset set to 0 and length set to size of the object.
+		// Fetching the entire object.
+		// 	Test case - 8.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], true, bytesData[0].byteData, nil},
+		// Test case with content-range 1 to objectSize .
+		// Test case - 9.
+		{bucketName, objectName, 1, int64(len(bytesData[0].byteData) - 1), buffers[1], buffers[1], true, bytesData[0].byteData[1:], nil},
+		// Test case with content-range 100 to objectSize - 100.
+		// Test case - 10.
+		{bucketName, objectName, 100, int64(len(bytesData[0].byteData) - 200), buffers[1], buffers[1], true,
+			bytesData[0].byteData[100 : len(bytesData[0].byteData)-100], nil},
+		// Test case with offset greater than the size of the object
+		// Test case - 11.
+		{bucketName, objectName, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), buffers[0],
+			NewEOFWriter(buffers[0], 100), false, []byte{},
+			InvalidRange{int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}},
+		// Test case with offset greater than the size of the object.
+		// Test case - 12.
+		{bucketName, objectName, -1, int64(len(bytesData[0].byteData)), buffers[0], new(bytes.Buffer), false, []byte{}, errUnexpected},
+		// Test case length parameter is more than the object size.
+		// Test case - 13.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData) + 1), buffers[1], buffers[1], false, bytesData[0].byteData,
+			InvalidRange{0, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData))}},
+		// Test case with `length` parameter set to a negative value.
+		// Test case - 14.
+		{bucketName, objectName, 0, int64(-1), buffers[1], buffers[1], false, bytesData[0].byteData, errUnexpected},
+		// Test case with offset + length > objectSize parameter set to a negative value.
+		// Test case - 15.
+		{bucketName, objectName, 2, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], false, bytesData[0].byteData,
+			InvalidRange{2, int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}},
+		// Test case with the writer set to nil.
+		// Test case - 16.
+		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], nil, false, bytesData[0].byteData, errUnexpected},
+	}
+
+	for i, testCase := range testCases {
+		err = obj.GetObject(testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead.", i+1, instanceType, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if !strings.Contains(err.Error(), testCase.err.Error()) {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.err.Error(), err.Error())
+			}
+		}
+		// Since there are cases for which GetObject fails, this is
+		// necessary. Test passes as expected, but the output values
+		// are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if !bytes.Equal(testCase.expectedData, testCase.getObjectData.Bytes()) {
+				t.Errorf("Test %d: %s: Data Mismatch: Expected data and the fetched data from GetObject doesn't match.", i+1, instanceType)
+			}
+			// empty the buffer so that it can be used to further cases.
+			testCase.getObjectData.Reset()
+		}
+	}
+}
+
+// Benchmarks for ObjectLayer.GetObject().
+// The intent is to benchmark GetObject for various sizes ranging from few bytes to 100MB.
+// Also each of these Benchmarks are run both XL and FS backends.
+
+// BenchmarkGetObjectVerySmallFS - Benchmark FS.GetObject() for object size of 10 bytes.
+func BenchmarkGetObjectVerySmallFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", 10)
+}
+
+// BenchmarkGetObjectVerySmallXL - Benchmark XL.GetObject() for object size of 10 bytes.
+func BenchmarkGetObjectVerySmallXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", 10)
+}
+
+// BenchmarkGetObject10KbFS - Benchmark FS.GetObject() for object size of 10KB.
+func BenchmarkGetObject10KbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", 10*1024)
+}
+
+// BenchmarkGetObject10KbXL - Benchmark XL.GetObject() for object size of 10KB.
+func BenchmarkGetObject10KbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", 10*1024)
+}
+
+// BenchmarkGetObject100KbFS - Benchmark FS.GetObject() for object size of 100KB.
+func BenchmarkGetObject100KbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", 100*1024)
+}
+
+// BenchmarkGetObject100KbXL - Benchmark XL.GetObject() for object size of 100KB.
+func BenchmarkGetObject100KbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", 100*1024)
+}
+
+// BenchmarkGetObject1MbFS - Benchmark FS.GetObject() for object size of 1MB.
+func BenchmarkGetObject1MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", 1024*1024)
+}
+
+// BenchmarkGetObject1MbXL - Benchmark XL.GetObject() for object size of 1MB.
+func BenchmarkGetObject1MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", 1024*1024)
+}
+
+// BenchmarkGetObject5MbFS - Benchmark FS.GetObject() for object size of 5MB.
+func BenchmarkGetObject5MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", 5*1024*1024)
+}
+
+// BenchmarkGetObject5MbXL - Benchmark XL.GetObject() for object size of 5MB.
+func BenchmarkGetObject5MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", 5*1024*1024)
+}
+
+// BenchmarkGetObject10MbFS - Benchmark FS.GetObject() for object size of 10MB.
+func BenchmarkGetObject10MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", 10*1024*1024)
+}
+
+// BenchmarkGetObject10MbXL - Benchmark XL.GetObject() for object size of 10MB.
+func BenchmarkGetObject10MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", 10*1024*1024)
+}
+
+// BenchmarkGetObject25MbFS - Benchmark FS.GetObject() for object size of 25MB.
+func BenchmarkGetObject25MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", 25*1024*1024)
+
+}
+
+// BenchmarkGetObject25MbXL - Benchmark XL.GetObject() for object size of 25MB.
+func BenchmarkGetObject25MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", 25*1024*1024)
+}
+
+// BenchmarkGetObject50MbFS - Benchmark FS.GetObject() for object size of 50MB.
+func BenchmarkGetObject50MbFS(b *testing.B) {
+	benchmarkGetObject(b, "FS", 50*1024*1024)
+}
+
+// BenchmarkGetObject50MbXL - Benchmark XL.GetObject() for object size of 50MB.
+func BenchmarkGetObject50MbXL(b *testing.B) {
+	benchmarkGetObject(b, "XL", 50*1024*1024)
+}
+
+// parallel benchmarks for ObjectLayer.GetObject() .
+
+// BenchmarkGetObjectParallelVerySmallFS - Benchmark FS.GetObject() for object size of 10 bytes.
+func BenchmarkGetObjectParallelVerySmallFS(b *testing.B) {
+	benchmarkGetObjectParallel(b, "FS", 10)
+}
+
+// BenchmarkGetObjectParallelVerySmallXL - Benchmark XL.GetObject() for object size of 10 bytes.
+func BenchmarkGetObjectParallelVerySmallXL(b *testing.B) {
+	benchmarkGetObjectParallel(b, "XL", 10)
+}
+
+// BenchmarkGetObjectParallel10KbFS - Benchmark FS.GetObject() for object size of 10KB.
+func BenchmarkGetObjectParallel10KbFS(b *testing.B) {
+	benchmarkGetObjectParallel(b, "FS", 10*1024)
+}
+
+// BenchmarkGetObjectParallel10KbXL - Benchmark XL.GetObject() for object size of 10KB.
+func BenchmarkGetObjectParallel10KbXL(b *testing.B) {
+	benchmarkGetObjectParallel(b, "XL", 10*1024)
+}
+
+// BenchmarkGetObjectParallel100KbFS - Benchmark FS.GetObject() for object size of 100KB.
+func BenchmarkGetObjectParallel100KbFS(b *testing.B) {
+	benchmarkGetObjectParallel(b, "FS", 100*1024)
+}
+
+// BenchmarkGetObjectParallel100KbXL - Benchmark XL.GetObject() for object size of 100KB.
+func BenchmarkGetObjectParallel100KbXL(b *testing.B) {
+	benchmarkGetObjectParallel(b, "XL", 100*1024)
+}
+
+// BenchmarkGetObjectParallel1MbFS - Benchmark FS.GetObject() for object size of 1MB.
+func BenchmarkGetObjectParallel1MbFS(b *testing.B) {
+	benchmarkGetObjectParallel(b, "FS", 1024*1024)
+}
+
+// BenchmarkGetObjectParallel1MbXL - Benchmark XL.GetObject() for object size of 1MB.
+func BenchmarkGetObjectParallel1MbXL(b *testing.B) {
+	benchmarkGetObjectParallel(b, "XL", 1024*1024)
+}
+
+// BenchmarkGetObjectParallel5MbFS - Benchmark FS.GetObject() for object size of 5MB.
+func BenchmarkGetObjectParallel5MbFS(b *testing.B) {
+	benchmarkGetObjectParallel(b, "FS", 5*1024*1024)
+}
+
+// BenchmarkGetObjectParallel5MbXL - Benchmark XL.GetObject() for object size of 5MB.
+func BenchmarkGetObjectParallel5MbXL(b *testing.B) {
+	benchmarkGetObjectParallel(b, "XL", 5*1024*1024)
+}
+
+// BenchmarkGetObjectParallel10MbFS - Benchmark FS.GetObject() for object size of 10MB.
+func BenchmarkGetObjectParallel10MbFS(b *testing.B) {
+	benchmarkGetObjectParallel(b, "FS", 10*1024*1024)
+}
+
+// BenchmarkGetObjectParallel10MbXL - Benchmark XL.GetObject() for object size of 10MB.
+func BenchmarkGetObjectParallel10MbXL(b *testing.B) {
+	benchmarkGetObjectParallel(b, "XL", 10*1024*1024)
+}
+
+// BenchmarkGetObjectParallel25MbFS - Benchmark FS.GetObject() for object size of 25MB.
+func BenchmarkGetObjectParallel25MbFS(b *testing.B) {
+	benchmarkGetObjectParallel(b, "FS", 25*1024*1024)
+
+}
+
+// BenchmarkGetObjectParallel25MbXL - Benchmark XL.GetObject() for object size of 25MB.
+func BenchmarkGetObjectParallel25MbXL(b *testing.B) {
+	benchmarkGetObjectParallel(b, "XL", 25*1024*1024)
+}
+
+// BenchmarkGetObjectParallel50MbFS - Benchmark FS.GetObject() for object size of 50MB.
+func BenchmarkGetObjectParallel50MbFS(b *testing.B) {
+	benchmarkGetObjectParallel(b, "FS", 50*1024*1024)
+}
+
+// BenchmarkGetObjectParallel50MbXL - Benchmark XL.GetObject() for object size of 50MB.
+func BenchmarkGetObjectParallel50MbXL(b *testing.B) {
+	benchmarkGetObjectParallel(b, "XL", 50*1024*1024)
+}

cmd/object-api-getobjectinfo_test.go

@@ -0,0 +1,106 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+)
+
+// Wrapper for calling GetObjectInfo tests for both XL multiple disks and single node setup.
+func TestGetObjectInfo(t *testing.T) {
+	ExecObjectLayerTest(t, testGetObjectInfo)
+}
+
+// Testing GetObjectInfo().
+func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// This bucket is used for testing getObjectInfo operations.
+	err := obj.MakeBucket("test-getobjectinfo")
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+	_, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+	resultCases := []ObjectInfo{
+		// ObjectInfo -1.
+		// ObjectName set to a existing object in the test case (Test case 14).
+		{Bucket: "test-getobjectinfo", Name: "Asia/asiapics.jpg", ContentType: "image/jpeg", IsDir: false},
+	}
+	testCases := []struct {
+		bucketName string
+		objectName string
+
+		// Expected output of GetObjectInfo.
+		result ObjectInfo
+		err    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names ( Test number 1-4 ).
+		{".test", "", ObjectInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Test cases with valid but non-existing bucket names (Test number 5-7).
+		{"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false},
+		{"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false},
+		// Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9).
+		{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
+		{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
+		// Test cases with non-existing object name with existing bucket (Test number 10-12).
+		{"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false},
+		{"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false},
+		{"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false},
+		// Test case with existing bucket but object name set to a directory (Test number 13).
+		{"test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false},
+		// Valid case with existing object (Test number 14).
+		{"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true},
+	}
+	for i, testCase := range testCases {
+		result, err := obj.GetObjectInfo(testCase.bucketName, testCase.objectName)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if testCase.err.Error() != err.Error() {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error())
+			}
+		}
+
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.result.Bucket != result.Bucket {
+				t.Fatalf("Test %d: %s: Expected Bucket name to be '%s', but found '%s' instead", i+1, instanceType, testCase.result.Bucket, result.Bucket)
+			}
+			if testCase.result.Name != result.Name {
+				t.Errorf("Test %d: %s: Expected Object name to be %s, but instead found it to be %s", i+1, instanceType, testCase.result.Name, result.Name)
+			}
+			if testCase.result.ContentType != result.ContentType {
+				t.Errorf("Test %d: %s: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.ContentType, result.ContentType)
+			}
+			if testCase.result.IsDir != result.IsDir {
+				t.Errorf("Test %d: %s: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsDir, result.IsDir)
+			}
+		}
+	}
+}

cmd/object-api-listobjects_test.go

@@ -0,0 +1,600 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Wrapper for calling ListObjects tests for both XL multiple disks and single node setup.
+func TestListObjects(t *testing.T) {
+	ExecObjectLayerTest(t, testListObjects)
+}
+
+// Unit test for ListObjects in general.
+func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
+
+	testBuckets := []string{
+		// This bucket is used for testing ListObject operations.
+		"test-bucket-list-object",
+		// Will not store any objects in this bucket,
+		// Its to test ListObjects on an empty bucket.
+		"empty-bucket",
+	}
+	for _, bucket := range testBuckets {
+		err := obj.MakeBucket(bucket)
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+	}
+
+	var err error
+	testObjects := []struct {
+		name    string
+		content string
+	}{
+		{"Asia-maps", "asis-maps"},
+		{"Asia/India/India-summer-photos-1", "contentstring"},
+		{"Asia/India/Karnataka/Bangalore/Koramangala/pics", "contentstring"},
+		{"newPrefix0", "newPrefix0"},
+		{"newPrefix1", "newPrefix1"},
+		{"newzen/zen/recurse/again/again/again/pics", "recurse"},
+		{"obj0", "obj0"},
+		{"obj1", "obj1"},
+		{"obj2", "obj2"},
+	}
+	for _, object := range testObjects {
+		_, err = obj.PutObject(testBuckets[0], object.name, int64(len(object.content)), bytes.NewBufferString(object.content), nil)
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+
+	}
+
+	// Formualting the result data set to be expected from ListObjects call inside the tests,
+	// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
+
+	resultCases := []ListObjectsInfo{
+		// ListObjectsResult-0.
+		// Testing for listing all objects in the bucket, (testCase 20,21,22).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-1.
+		// Used for asserting the truncated case, (testCase 23).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+			},
+		},
+		// ListObjectsResult-2.
+		// (TestCase 24).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+			},
+		},
+		// ListObjectsResult-3.
+		// (TestCase 25).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+		// ListObjectsResult-4.
+		// Again used for truncated case.
+		// (TestCase 26).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+			},
+		},
+		// ListObjectsResult-5.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix "new", (testCase 27-29).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-6.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix = "obj", (testCase 30).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-7.
+		// Used for Asserting prefixes and truncation.
+		// Used for test case with prefix = "new" and maxKeys = 1, (testCase 31).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+			},
+		},
+		// ListObjectsResult-8.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "obj0"},
+				{Name: "obj1"},
+			},
+		},
+		// ListObjectsResult-9.
+		// Used for asserting the case with marker, but without prefix.
+		//marker is set to "newPrefix0" in the testCase, (testCase 33).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-10.
+		//marker is set to "newPrefix1" in the testCase, (testCase 34).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-11.
+		//marker is set to "obj0" in the testCase, (testCase 35).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-12.
+		// Marker is set to "obj1" in the testCase, (testCase 36).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-13.
+		// Marker is set to "man" in the testCase, (testCase37).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-14.
+		// Marker is set to "Abc" in the testCase, (testCase 39).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-15.
+		// Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-16.
+		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-17.
+		// Used for asserting the case with marker, without prefix but with truncation.
+		// Marker =  "newPrefix0" & maxKeys = 3 in the testCase, (testCase42).
+		// Output truncated to 3 values.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+			},
+		},
+		// ListObjectsResult-18.
+		// Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43).
+		// Output truncated to 1 value.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-19.
+		// Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44).
+		// Output truncated to 1 value.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+			},
+		},
+		// ListObjectsResult-20.
+		// Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-21.
+		// Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-22.
+		// Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-23.
+		// Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+
+		// ListObjectsResult-24.
+		// Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+
+		// ListObjectsResult-25.
+		// Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+			},
+		},
+		// ListObjectsResult-26.
+		// prefix = "new" and delimiter is set in the testCase.(testCase 58).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+			},
+		},
+		// ListObjectsResult-27.
+		// Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/India-summer-photos-1"},
+			},
+		},
+		// ListObjectsResult-28.
+		// Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-29.
+		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimeter set, (testCase 61).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-30.
+		// Prefix and Delimiter is set to '/', (testCase 62).
+		{
+			IsTruncated: false,
+			Objects:     []ObjectInfo{},
+		},
+	}
+
+	testCases := []struct {
+		// Inputs to ListObjects.
+		bucketName string
+		prefix     string
+		marker     string
+		delimeter  string
+		maxKeys    int
+		// Expected output of ListObjects.
+		result ListObjectsInfo
+		err    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names ( Test number 1-4 ).
+		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Using an existing file for bucket name, but its not a directory (5).
+		{"simple-file.txt", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "simple-file.txt"}, false},
+		// Valid bucket names, but they donot exist (6-8).
+		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		// Valid, existing bucket, but sending invalid delimeter values (9-10).
+		// Empty string < "" > and forward slash < / > are the ony two valid arguments for delimeter.
+		{"test-bucket-list-object", "", "", "*", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "*"), false},
+		{"test-bucket-list-object", "", "", "-", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "-"), false},
+		// Testing for failure cases with both perfix and marker (11).
+		// The prefix and marker combination to be valid it should satisy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
+		// Setting a non-existing directory to be prefix (12-13).
+		{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
+		{"empty-bucket", "europe/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
+		// Testing on empty bucket, that is, bucket without any objects in it (14).
+		{"empty-bucket", "", "", "", 0, ListObjectsInfo{}, nil, true},
+		// Setting maxKeys to negative value (15-16).
+		{"empty-bucket", "", "", "", -1, ListObjectsInfo{}, nil, true},
+		{"empty-bucket", "", "", "", 1, ListObjectsInfo{}, nil, true},
+		// Setting maxKeys to a very large value (17).
+		{"empty-bucket", "", "", "", 1111000000000000, ListObjectsInfo{}, nil, true},
+		// Testing for all 7 objects in the bucket (18).
+		{"test-bucket-list-object", "", "", "", 9, resultCases[0], nil, true},
+		//Testing for negative value of maxKey, this should set maxKeys to listObjectsLimit (19).
+		{"test-bucket-list-object", "", "", "", -1, resultCases[0], nil, true},
+		// Testing for very large value of maxKey, this should set maxKeys to listObjectsLimit (20).
+		{"test-bucket-list-object", "", "", "", 1234567891011, resultCases[0], nil, true},
+		// Testing for trancated value (21-24).
+		{"test-bucket-list-object", "", "", "", 5, resultCases[1], nil, true},
+		{"test-bucket-list-object", "", "", "", 4, resultCases[2], nil, true},
+		{"test-bucket-list-object", "", "", "", 3, resultCases[3], nil, true},
+		{"test-bucket-list-object", "", "", "", 1, resultCases[4], nil, true},
+		// Testing with prefix (25-28).
+		{"test-bucket-list-object", "new", "", "", 3, resultCases[5], nil, true},
+		{"test-bucket-list-object", "new", "", "", 4, resultCases[5], nil, true},
+		{"test-bucket-list-object", "new", "", "", 5, resultCases[5], nil, true},
+		{"test-bucket-list-object", "obj", "", "", 3, resultCases[6], nil, true},
+		// Testing with prefix and truncation (29-30).
+		{"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true},
+		{"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true},
+		// Testing with marker, but without prefix and truncation (31-35).
+		{"test-bucket-list-object", "", "newPrefix0", "", 5, resultCases[9], nil, true},
+		{"test-bucket-list-object", "", "newPrefix1", "", 4, resultCases[10], nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 2, resultCases[11], nil, true},
+		{"test-bucket-list-object", "", "obj1", "", 1, resultCases[12], nil, true},
+		{"test-bucket-list-object", "", "man", "", 10, resultCases[13], nil, true},
+		// Marker being set to a value which is greater than and all object names when sorted (36).
+		// Expected to send an empty response in this case.
+		{"test-bucket-list-object", "", "zen", "", 10, ListObjectsInfo{}, nil, true},
+		// Marker being set to a value which is lesser than and all object names when sorted (37).
+		// Expected to send all the objects in the bucket in this case.
+		{"test-bucket-list-object", "", "Abc", "", 10, resultCases[14], nil, true},
+		// Marker is to a hierarhical value (38-39).
+		{"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "", 10, resultCases[15], nil, true},
+		{"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", 10, resultCases[16], nil, true},
+		// Testing with marker and truncation, but no prefix (40-42).
+		{"test-bucket-list-object", "", "newPrefix0", "", 3, resultCases[17], nil, true},
+		{"test-bucket-list-object", "", "newPrefix1", "", 1, resultCases[18], nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 1, resultCases[19], nil, true},
+		// Testing with both marker and prefix, but without truncation (43-45).
+		// The valid combination of marker and prefix should satisfy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "obj", "obj0", "", 2, resultCases[20], nil, true},
+		{"test-bucket-list-object", "obj", "obj1", "", 1, resultCases[21], nil, true},
+		{"test-bucket-list-object", "new", "newPrefix0", "", 2, resultCases[22], nil, true},
+		// Testing with maxKeys set to 0 (46-52).
+		// The parameters have to valid.
+		{"test-bucket-list-object", "", "obj1", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "new", "", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "obj0", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "obj1", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "new", "newPrefix0", "", 0, ListObjectsInfo{}, nil, true},
+		// Tests on hierarchical key names as prefix.
+		// Without delimteter the code should recurse into the prefix Dir.
+		// Tests with prefix, but without delimiter (53-54).
+		{"test-bucket-list-object", "Asia/India/", "", "", 10, resultCases[23], nil, true},
+		{"test-bucket-list-object", "Asia", "", "", 10, resultCases[24], nil, true},
+		// Tests with prefix and delimiter (55-57).
+		// With delimeter the code should not recurse into the sub-directories of prefix Dir.
+		{"test-bucket-list-object", "Asia", "", "/", 10, resultCases[25], nil, true},
+		{"test-bucket-list-object", "new", "", "/", 10, resultCases[26], nil, true},
+		{"test-bucket-list-object", "Asia/India/", "", "/", 10, resultCases[27], nil, true},
+		// Test with marker set as hierarhical value and with delimiter. (58-59)
+		{"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "/", 10, resultCases[28], nil, true},
+		{"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "/", 10, resultCases[29], nil, true},
+		// Test with prefix and delimiter set to '/'. (60)
+		{"test-bucket-list-object", "/", "", "/", 10, resultCases[30], nil, true},
+	}
+
+	for i, testCase := range testCases {
+		result, err := obj.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if !strings.Contains(err.Error(), testCase.err.Error()) {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error())
+			}
+		}
+		// Since there are cases for which ListObjects fails, this is
+		// necessary. Test passes as expected, but the output values
+		// are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			// The length of the expected ListObjectsResult.Objects
+			// should match in both expected result from test cases
+			// and in the output. On failure calling t.Fatalf,
+			// otherwise it may lead to index out of range error in
+			// assertion following this.
+			if len(testCase.result.Objects) != len(result.Objects) {
+				t.Fatalf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.result.Objects), len(result.Objects))
+			}
+			for j := 0; j < len(testCase.result.Objects); j++ {
+				if testCase.result.Objects[j].Name != result.Objects[j].Name {
+					t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name)
+				}
+			}
+			if testCase.result.IsTruncated != result.IsTruncated {
+				t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsTruncated, result.IsTruncated)
+			}
+
+		}
+		// Take ListObject treeWalk go-routine to completion, if available in the treewalk pool.
+		if result.IsTruncated {
+			_, err = obj.ListObjects(testCase.bucketName, testCase.prefix, result.NextMarker, testCase.delimeter, 1000)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkListObjects(b *testing.B) {
+	// Make a temporary directory to use as the obj.
+	directory, err := ioutil.TempDir("", "minio-list-benchmark")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer removeAll(directory)
+
+	// Create the obj.
+	obj, err := newFSObjects(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Create a bucket.
+	err = obj.MakeBucket("ls-benchmark-bucket")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	for i := 0; i < 20000; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = obj.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+
+	// List the buckets over and over and over.
+	for i := 0; i < b.N; i++ {
+		_, err = obj.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/object-api-putobject_test.go

@@ -0,0 +1,553 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+// md5Hex ignores error from Write method since it never returns one. Check
+// crypto/md5 doc for more details.
+func md5Hex(b []byte) string {
+	md5Writer := md5.New()
+	md5Writer.Write(b)
+	return hex.EncodeToString(md5Writer.Sum(nil))
+}
+
+func md5Header(data []byte) map[string]string {
+	return map[string]string{"md5Sum": md5Hex([]byte(data))}
+}
+
+// Wrapper for calling PutObject tests for both XL multiple disks and single node setup.
+func TestObjectAPIPutObject(t *testing.T) {
+	ExecObjectLayerTest(t, testObjectAPIPutObject)
+}
+
+// Tests validate correctness of PutObject.
+func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// Generating cases for which the PutObject fails.
+	bucket := "minio-bucket"
+	object := "minio-object"
+
+	// Create bucket.
+	err := obj.MakeBucket(bucket)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Creating a dummy bucket for tests.
+	err = obj.MakeBucket("unused-bucket")
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	var (
+		nilBytes    []byte
+		data        = []byte("hello")
+		fiveMBBytes = bytes.Repeat([]byte("a"), 5*1024*124)
+	)
+	invalidMD5 := md5Hex([]byte("meh"))
+	invalidMD5Header := md5Header([]byte("meh"))
+
+	testCases := []struct {
+		bucketName     string
+		objName        string
+		inputData      []byte
+		inputMeta      map[string]string
+		intputDataSize int64
+		// expected error output.
+		expectedMd5   string
+		expectedError error
+	}{
+		// Test case  1-4.
+		// Cases with invalid bucket name.
+		{".test", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: ".test"}},
+		{"------", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: "------"}},
+		{"$this-is-not-valid-too", "obj", []byte(""), nil, 0, "",
+			BucketNameInvalid{Bucket: "$this-is-not-valid-too"}},
+		{"a", "obj", []byte(""), nil, 0, "", BucketNameInvalid{Bucket: "a"}},
+
+		// Test case - 5.
+		// Case with invalid object names.
+		{bucket, "", []byte(""), nil, 0, "", ObjectNameInvalid{Bucket: bucket, Object: ""}},
+
+		// Test case - 6.
+		// Valid object and bucket names but non-existent bucket.
+		{"abc", "def", []byte(""), nil, 0, "", BucketNotFound{Bucket: "abc"}},
+
+		// Test case - 7.
+		// Input to replicate Md5 mismatch.
+		{bucket, object, []byte(""), map[string]string{"md5Sum": "a35"}, 0, "",
+			BadDigest{ExpectedMD5: "a35", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"}},
+
+		// Test case - 8.
+		// Input with size more than the size of actual data inside the reader.
+		{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, int64(len("abcd") + 1), "",
+			IncompleteBody{}},
+
+		// Test case - 9.
+		// Input with size less than the size of actual data inside the reader.
+		{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "a35"}, int64(len("abcd") - 1), "",
+			BadDigest{ExpectedMD5: "a35", CalculatedMD5: "900150983cd24fb0d6963f7d28e17f72"}},
+
+		// Test case - 10-13.
+		// Validating for success cases.
+		{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), "", nil},
+		{bucket, object, []byte("efgh"), map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), "", nil},
+		{bucket, object, []byte("ijkl"), map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), "", nil},
+		{bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), "", nil},
+
+		// Test case 14-16.
+		// With no metadata
+		{bucket, object, data, nil, int64(len(data)), md5Hex(data), nil},
+		{bucket, object, nilBytes, nil, int64(len(nilBytes)), md5Hex(nilBytes), nil},
+		{bucket, object, fiveMBBytes, nil, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil},
+
+		// Test case 17-19.
+		// With arbitrary metadata
+		{bucket, object, data, map[string]string{"answer": "42"}, int64(len(data)), md5Hex(data), nil},
+		{bucket, object, nilBytes, map[string]string{"answer": "42"}, int64(len(nilBytes)), md5Hex(nilBytes), nil},
+		{bucket, object, fiveMBBytes, map[string]string{"answer": "42"}, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil},
+
+		// Test case 20-22.
+		// With valid md5sum in header
+		{bucket, object, data, md5Header(data), int64(len(data)), md5Hex(data), nil},
+		{bucket, object, nilBytes, md5Header(nilBytes), int64(len(nilBytes)), md5Hex(nilBytes), nil},
+		{bucket, object, fiveMBBytes, md5Header(fiveMBBytes), int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), nil},
+
+		// Test case 23-25.
+		// data with invalid md5sum in header
+		{bucket, object, data, invalidMD5Header, int64(len(data)), md5Hex(data), BadDigest{invalidMD5, md5Hex(data)}},
+		{bucket, object, nilBytes, invalidMD5Header, int64(len(nilBytes)), md5Hex(nilBytes), BadDigest{invalidMD5, md5Hex(nilBytes)}},
+		{bucket, object, fiveMBBytes, invalidMD5Header, int64(len(fiveMBBytes)), md5Hex(fiveMBBytes), BadDigest{invalidMD5, md5Hex(fiveMBBytes)}},
+
+		// Test case 26-28.
+		// data with size different from the actual number of bytes available in the reader
+		{bucket, object, data, nil, int64(len(data) - 1), md5Hex(data[:len(data)-1]), nil},
+		{bucket, object, nilBytes, nil, int64(len(nilBytes) + 1), md5Hex(nilBytes), IncompleteBody{}},
+		{bucket, object, fiveMBBytes, nil, int64(0), md5Hex(fiveMBBytes), nil},
+	}
+
+	for i, testCase := range testCases {
+		actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta)
+		if actualErr != nil && testCase.expectedError == nil {
+			t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error())
+		}
+		if actualErr == nil && testCase.expectedError != nil {
+			t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && testCase.expectedError != actualErr {
+			t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error())
+		}
+		// Test passes as expected, but the output values are verified for correctness here.
+		if actualErr == nil {
+			// Asserting whether the md5 output is correct.
+			if expectedMD5, ok := testCase.inputMeta["md5Sum"]; ok && expectedMD5 != actualMd5Hex {
+				t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualMd5Hex)
+			}
+		}
+	}
+}
+
+// Wrapper for calling PutObject tests for both XL multiple disks case
+// when quorum is not available.
+func TestObjectAPIPutObjectDiskNotFound(t *testing.T) {
+	ExecObjectLayerDiskNotFoundTest(t, testObjectAPIPutObjectDiskNotFOund)
+}
+
+// Tests validate correctness of PutObject.
+func testObjectAPIPutObjectDiskNotFOund(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
+	// Generating cases for which the PutObject fails.
+	bucket := "minio-bucket"
+	object := "minio-object"
+
+	// Create bucket.
+	err := obj.MakeBucket(bucket)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Creating a dummy bucket for tests.
+	err = obj.MakeBucket("unused-bucket")
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Take 8 disks down, one more we loose quorum on 16 disk node.
+	for _, disk := range disks[:7] {
+		removeAll(disk)
+	}
+
+	testCases := []struct {
+		bucketName     string
+		objName        string
+		inputData      []byte
+		inputMeta      map[string]string
+		intputDataSize int64
+		// flag indicating whether the test should pass.
+		shouldPass bool
+		// expected error output.
+		expectedMd5   string
+		expectedError error
+	}{
+		// Validating for success cases.
+		{bucket, object, []byte("abcd"), map[string]string{"md5Sum": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), true, "", nil},
+		{bucket, object, []byte("efgh"), map[string]string{"md5Sum": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), true, "", nil},
+		{bucket, object, []byte("ijkl"), map[string]string{"md5Sum": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), true, "", nil},
+		{bucket, object, []byte("mnop"), map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), true, "", nil},
+	}
+
+	for i, testCase := range testCases {
+		actualMd5Hex, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if testCase.expectedError.Error() != actualErr.Error() {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1,
+					instanceType, testCase.expectedError.Error(), actualErr.Error())
+			}
+		}
+		// Test passes as expected, but the output values are verified for correctness here.
+		if actualErr == nil && testCase.shouldPass {
+			// Asserting whether the md5 output is correct.
+			if testCase.inputMeta["md5Sum"] != actualMd5Hex {
+				t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualMd5Hex)
+			}
+		}
+	}
+
+	// This causes quorum failure verify.
+	removeAll(disks[len(disks)-1])
+
+	// Validate the last test.
+	testCase := struct {
+		bucketName     string
+		objName        string
+		inputData      []byte
+		inputMeta      map[string]string
+		intputDataSize int64
+		// flag indicating whether the test should pass.
+		shouldPass bool
+		// expected error output.
+		expectedMd5   string
+		expectedError error
+	}{
+		bucket,
+		object,
+		[]byte("mnop"),
+		map[string]string{"md5Sum": "e132e96a5ddad6da8b07bba6f6131fef"},
+		int64(len("mnop")),
+		false,
+		"",
+		InsufficientWriteQuorum{},
+	}
+	_, actualErr := obj.PutObject(testCase.bucketName, testCase.objName, testCase.intputDataSize, bytes.NewReader(testCase.inputData), testCase.inputMeta)
+	if actualErr != nil && testCase.shouldPass {
+		t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", len(testCases)+1, instanceType, actualErr.Error())
+	}
+	// Failed as expected, but does it fail for the expected reason.
+	if actualErr != nil && !testCase.shouldPass {
+		if testCase.expectedError.Error() != actualErr.Error() {
+			t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", len(testCases)+1, instanceType, testCase.expectedError.Error(), actualErr.Error())
+		}
+	}
+}
+
+// Wrapper for calling PutObject tests for both XL multiple disks and single node setup.
+func TestObjectAPIPutObjectStaleFiles(t *testing.T) {
+	ExecObjectLayerStaleFilesTest(t, testObjectAPIPutObjectStaleFiles)
+}
+
+// Tests validate correctness of PutObject.
+func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
+	// Generating cases for which the PutObject fails.
+	bucket := "minio-bucket"
+	object := "minio-object"
+
+	// Create bucket.
+	err := obj.MakeBucket(bucket)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	data := []byte("hello, world")
+	// Create object.
+	_, err = obj.PutObject(bucket, object, int64(len(data)), bytes.NewReader(data), nil)
+	if err != nil {
+		// Failed to create object, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	for _, disk := range disks {
+		tmpMetaDir := path.Join(disk, minioMetaBucket, tmpMetaPrefix)
+		if !isDirEmpty(tmpMetaDir) {
+			t.Fatalf("%s: expected: empty, got: non-empty", tmpMetaDir)
+		}
+	}
+}
+
+// Wrapper for calling Multipart PutObject tests for both XL multiple disks and single node setup.
+func TestObjectAPIMultipartPutObjectStaleFiles(t *testing.T) {
+	ExecObjectLayerStaleFilesTest(t, testObjectAPIMultipartPutObjectStaleFiles)
+}
+
+// Tests validate correctness of PutObject.
+func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
+	// Generating cases for which the PutObject fails.
+	bucket := "minio-bucket"
+	object := "minio-object"
+
+	// Create bucket.
+	err := obj.MakeBucket(bucket)
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Initiate Multipart Upload on the above created bucket.
+	uploadID, err := obj.NewMultipartUpload(bucket, object, nil)
+	if err != nil {
+		// Failed to create NewMultipartUpload, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Upload part1.
+	fiveMBBytes := bytes.Repeat([]byte("a"), 5*1024*1024)
+	md5Writer := md5.New()
+	md5Writer.Write(fiveMBBytes)
+	etag1 := hex.EncodeToString(md5Writer.Sum(nil))
+	_, err = obj.PutObjectPart(bucket, object, uploadID, 1, int64(len(fiveMBBytes)), bytes.NewReader(fiveMBBytes), etag1)
+	if err != nil {
+		// Failed to upload object part, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Upload part2.
+	data := []byte("hello, world")
+	md5Writer = md5.New()
+	md5Writer.Write(data)
+	etag2 := hex.EncodeToString(md5Writer.Sum(nil))
+	_, err = obj.PutObjectPart(bucket, object, uploadID, 2, int64(len(data)), bytes.NewReader(data), etag2)
+	if err != nil {
+		// Failed to upload object part, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	// Complete multipart.
+	parts := []completePart{
+		{ETag: etag1, PartNumber: 1},
+		{ETag: etag2, PartNumber: 2},
+	}
+	_, err = obj.CompleteMultipartUpload(bucket, object, uploadID, parts)
+	if err != nil {
+		// Failed to complete multipart upload, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	for _, disk := range disks {
+		tmpMetaDir := path.Join(disk, minioMetaBucket, tmpMetaPrefix)
+		files, err := ioutil.ReadDir(tmpMetaDir)
+		if err != nil {
+			// Its OK to have non-existen tmpMetaDir.
+			if os.IsNotExist(err) {
+				continue
+			}
+
+			// Print the error
+			t.Errorf("%s", err)
+		}
+
+		if len(files) != 0 {
+			t.Fatalf("%s: expected: empty, got: non-empty. content: %s", tmpMetaDir, files)
+		}
+	}
+}
+
+// Benchmarks for ObjectLayer.PutObject().
+// The intent is to benchmark PutObject for various sizes ranging from few bytes to 100MB.
+// Also each of these Benchmarks are run both XL and FS backends.
+
+// BenchmarkPutObjectVerySmallFS - Benchmark FS.PutObject() for object size of 10 bytes.
+func BenchmarkPutObjectVerySmallFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", 10)
+}
+
+// BenchmarkPutObjectVerySmallXL - Benchmark XL.PutObject() for object size of 10 bytes.
+func BenchmarkPutObjectVerySmallXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", 10)
+}
+
+// BenchmarkPutObject10KbFS - Benchmark FS.PutObject() for object size of 10KB.
+func BenchmarkPutObject10KbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", 10*1024)
+}
+
+// BenchmarkPutObject10KbXL - Benchmark XL.PutObject() for object size of 10KB.
+func BenchmarkPutObject10KbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", 10*1024)
+}
+
+// BenchmarkPutObject100KbFS - Benchmark FS.PutObject() for object size of 100KB.
+func BenchmarkPutObject100KbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", 100*1024)
+}
+
+// BenchmarkPutObject100KbXL - Benchmark XL.PutObject() for object size of 100KB.
+func BenchmarkPutObject100KbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", 100*1024)
+}
+
+// BenchmarkPutObject1MbFS - Benchmark FS.PutObject() for object size of 1MB.
+func BenchmarkPutObject1MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", 1024*1024)
+}
+
+// BenchmarkPutObject1MbXL - Benchmark XL.PutObject() for object size of 1MB.
+func BenchmarkPutObject1MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", 1024*1024)
+}
+
+// BenchmarkPutObject5MbFS - Benchmark FS.PutObject() for object size of 5MB.
+func BenchmarkPutObject5MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", 5*1024*1024)
+}
+
+// BenchmarkPutObject5MbXL - Benchmark XL.PutObject() for object size of 5MB.
+func BenchmarkPutObject5MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", 5*1024*1024)
+}
+
+// BenchmarkPutObject10MbFS - Benchmark FS.PutObject() for object size of 10MB.
+func BenchmarkPutObject10MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", 10*1024*1024)
+}
+
+// BenchmarkPutObject10MbXL - Benchmark XL.PutObject() for object size of 10MB.
+func BenchmarkPutObject10MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", 10*1024*1024)
+}
+
+// BenchmarkPutObject25MbFS - Benchmark FS.PutObject() for object size of 25MB.
+func BenchmarkPutObject25MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", 25*1024*1024)
+
+}
+
+// BenchmarkPutObject25MbXL - Benchmark XL.PutObject() for object size of 25MB.
+func BenchmarkPutObject25MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", 25*1024*1024)
+}
+
+// BenchmarkPutObject50MbFS - Benchmark FS.PutObject() for object size of 50MB.
+func BenchmarkPutObject50MbFS(b *testing.B) {
+	benchmarkPutObject(b, "FS", 50*1024*1024)
+}
+
+// BenchmarkPutObject50MbXL - Benchmark XL.PutObject() for object size of 50MB.
+func BenchmarkPutObject50MbXL(b *testing.B) {
+	benchmarkPutObject(b, "XL", 50*1024*1024)
+}
+
+// parallel benchmarks for ObjectLayer.PutObject() .
+
+// BenchmarkParallelPutObjectVerySmallFS - BenchmarkParallel FS.PutObject() for object size of 10 bytes.
+func BenchmarkParallelPutObjectVerySmallFS(b *testing.B) {
+	benchmarkPutObjectParallel(b, "FS", 10)
+}
+
+// BenchmarkParallelPutObjectVerySmallXL - BenchmarkParallel XL.PutObject() for object size of 10 bytes.
+func BenchmarkParallelPutObjectVerySmallXL(b *testing.B) {
+	benchmarkPutObjectParallel(b, "XL", 10)
+}
+
+// BenchmarkParallelPutObject10KbFS - BenchmarkParallel FS.PutObject() for object size of 10KB.
+func BenchmarkParallelPutObject10KbFS(b *testing.B) {
+	benchmarkPutObjectParallel(b, "FS", 10*1024)
+}
+
+// BenchmarkParallelPutObject10KbXL - BenchmarkParallel XL.PutObject() for object size of 10KB.
+func BenchmarkParallelPutObject10KbXL(b *testing.B) {
+	benchmarkPutObjectParallel(b, "XL", 10*1024)
+}
+
+// BenchmarkParallelPutObject100KbFS - BenchmarkParallel FS.PutObject() for object size of 100KB.
+func BenchmarkParallelPutObject100KbFS(b *testing.B) {
+	benchmarkPutObjectParallel(b, "FS", 100*1024)
+}
+
+// BenchmarkParallelPutObject100KbXL - BenchmarkParallel XL.PutObject() for object size of 100KB.
+func BenchmarkParallelPutObject100KbXL(b *testing.B) {
+	benchmarkPutObjectParallel(b, "XL", 100*1024)
+}
+
+// BenchmarkParallelPutObject1MbFS - BenchmarkParallel FS.PutObject() for object size of 1MB.
+func BenchmarkParallelPutObject1MbFS(b *testing.B) {
+	benchmarkPutObjectParallel(b, "FS", 1024*1024)
+}
+
+// BenchmarkParallelPutObject1MbXL - BenchmarkParallel XL.PutObject() for object size of 1MB.
+func BenchmarkParallelPutObject1MbXL(b *testing.B) {
+	benchmarkPutObjectParallel(b, "XL", 1024*1024)
+}
+
+// BenchmarkParallelPutObject5MbFS - BenchmarkParallel FS.PutObject() for object size of 5MB.
+func BenchmarkParallelPutObject5MbFS(b *testing.B) {
+	benchmarkPutObjectParallel(b, "FS", 5*1024*1024)
+}
+
+// BenchmarkParallelPutObject5MbXL - BenchmarkParallel XL.PutObject() for object size of 5MB.
+func BenchmarkParallelPutObject5MbXL(b *testing.B) {
+	benchmarkPutObjectParallel(b, "XL", 5*1024*1024)
+}
+
+// BenchmarkParallelPutObject10MbFS - BenchmarkParallel FS.PutObject() for object size of 10MB.
+func BenchmarkParallelPutObject10MbFS(b *testing.B) {
+	benchmarkPutObjectParallel(b, "FS", 10*1024*1024)
+}
+
+// BenchmarkParallelPutObject10MbXL - BenchmarkParallel XL.PutObject() for object size of 10MB.
+func BenchmarkParallelPutObject10MbXL(b *testing.B) {
+	benchmarkPutObjectParallel(b, "XL", 10*1024*1024)
+}
+
+// BenchmarkParallelPutObject25MbFS - BenchmarkParallel FS.PutObject() for object size of 25MB.
+func BenchmarkParallelPutObject25MbFS(b *testing.B) {
+	benchmarkPutObjectParallel(b, "FS", 25*1024*1024)
+
+}
+
+// BenchmarkParallelPutObject25MbXL - BenchmarkParallel XL.PutObject() for object size of 25MB.
+func BenchmarkParallelPutObject25MbXL(b *testing.B) {
+	benchmarkPutObjectParallel(b, "XL", 25*1024*1024)
+}

cmd/object-common.go

@@ -0,0 +1,187 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+const (
+	// Block size used for all internal operations version 1.
+	blockSizeV1 = 10 * 1024 * 1024 // 10MiB.
+
+	// Staging buffer read size for all internal operations version 1.
+	readSizeV1 = 128 * 1024 // 128KiB.
+
+	// Buckets meta prefix.
+	bucketMetaPrefix = "buckets"
+)
+
+// isErrIgnored should we ignore this error?, takes a list of errors which can be ignored.
+func isErrIgnored(err error, ignoredErrs []error) bool {
+	for _, ignoredErr := range ignoredErrs {
+		if ignoredErr == err {
+			return true
+		}
+	}
+	return false
+}
+
+// House keeping code needed for FS.
+func fsHouseKeeping(storageDisk StorageAPI) error {
+	// Cleanup all temp entries upon start.
+	err := cleanupDir(storageDisk, minioMetaBucket, tmpMetaPrefix)
+	if err != nil {
+		return toObjectErr(err, minioMetaBucket, tmpMetaPrefix)
+	}
+	return nil
+}
+
+// Depending on the disk type network or local, initialize storage API.
+func newStorageAPI(disk string) (storage StorageAPI, err error) {
+	if !strings.ContainsRune(disk, ':') || filepath.VolumeName(disk) != "" {
+		// Initialize filesystem storage API.
+		return newPosix(disk)
+	}
+	// Initialize rpc client storage API.
+	return newRPCClient(disk)
+}
+
+// Initializes meta volume on all input storage disks.
+func initMetaVolume(storageDisks []StorageAPI) error {
+	// This happens for the first time, but keep this here since this
+	// is the only place where it can be made expensive optimizing all
+	// other calls. Create minio meta volume, if it doesn't exist yet.
+	var wg = &sync.WaitGroup{}
+
+	// Initialize errs to collect errors inside go-routine.
+	var errs = make([]error, len(storageDisks))
+
+	// Initialize all disks in parallel.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			// Ignore create meta volume on disks which are not found.
+			continue
+		}
+		wg.Add(1)
+		go func(index int, disk StorageAPI) {
+			// Indicate this wait group is done.
+			defer wg.Done()
+
+			// Attempt to create `.minio`.
+			err := disk.MakeVol(minioMetaBucket)
+			if err != nil {
+				switch err {
+				// Ignored errors.
+				case errVolumeExists, errDiskNotFound, errFaultyDisk:
+				default:
+					errs[index] = err
+				}
+			}
+		}(index, disk)
+	}
+
+	// Wait for all cleanup to finish.
+	wg.Wait()
+
+	// Return upon first error.
+	for _, err := range errs {
+		if err == nil {
+			continue
+		}
+		return toObjectErr(err, minioMetaBucket)
+	}
+
+	// Return success here.
+	return nil
+}
+
+// House keeping code needed for XL.
+func xlHouseKeeping(storageDisks []StorageAPI) error {
+	// This happens for the first time, but keep this here since this
+	// is the only place where it can be made expensive optimizing all
+	// other calls. Create metavolume.
+	var wg = &sync.WaitGroup{}
+
+	// Initialize errs to collect errors inside go-routine.
+	var errs = make([]error, len(storageDisks))
+
+	// Initialize all disks in parallel.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int, disk StorageAPI) {
+			// Indicate this wait group is done.
+			defer wg.Done()
+
+			// Cleanup all temp entries upon start.
+			err := cleanupDir(disk, minioMetaBucket, tmpMetaPrefix)
+			if err != nil {
+				errs[index] = err
+			}
+		}(index, disk)
+	}
+
+	// Wait for all cleanup to finish.
+	wg.Wait()
+
+	// Return upon first error.
+	for _, err := range errs {
+		if err == nil {
+			continue
+		}
+		return toObjectErr(err, minioMetaBucket, tmpMetaPrefix)
+	}
+
+	// Return success here.
+	return nil
+}
+
+// Cleanup a directory recursively.
+func cleanupDir(storage StorageAPI, volume, dirPath string) error {
+	var delFunc func(string) error
+	// Function to delete entries recursively.
+	delFunc = func(entryPath string) error {
+		if !strings.HasSuffix(entryPath, slashSeparator) {
+			// Delete the file entry.
+			return storage.DeleteFile(volume, entryPath)
+		}
+
+		// If it's a directory, list and call delFunc() for each entry.
+		entries, err := storage.ListDir(volume, entryPath)
+		// If entryPath prefix never existed, safe to ignore.
+		if err == errFileNotFound {
+			return nil
+		} else if err != nil { // For any other errors fail.
+			return err
+		} // else on success..
+
+		// Recurse and delete all other entries.
+		for _, entry := range entries {
+			if err = delFunc(pathJoin(entryPath, entry)); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	err := delFunc(retainSlash(pathJoin(dirPath)))
+	return err
+}

cmd/object-datatypes.go

@@ -0,0 +1,224 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "time"
+
+// StorageInfo - represents total capacity of underlying storage.
+type StorageInfo struct {
+	// Total disk space.
+	Total int64
+	// Free available disk space.
+	Free int64
+}
+
+// BucketInfo - represents bucket metadata.
+type BucketInfo struct {
+	// Name of the bucket.
+	Name string
+
+	// Date and time when the bucket was created.
+	Created time.Time
+}
+
+// ObjectInfo - represents object metadata.
+type ObjectInfo struct {
+	// Name of the bucket.
+	Bucket string
+
+	// Name of the object.
+	Name string
+
+	// Date and time when the object was last modified.
+	ModTime time.Time
+
+	// Total object size.
+	Size int64
+
+	// IsDir indicates if the object is prefix.
+	IsDir bool
+
+	// Hex encoded md5 checksum of the object.
+	MD5Sum string
+
+	// A standard MIME type describing the format of the object.
+	ContentType string
+
+	// Specifies what content encodings have been applied to the object and thus
+	// what decoding mechanisms must be applied to obtain the object referenced
+	// by the Content-Type header field.
+	ContentEncoding string
+
+	// User-Defined metadata
+	UserDefined map[string]string
+}
+
+// ListPartsInfo - represents list of all parts.
+type ListPartsInfo struct {
+	// Name of the bucket.
+	Bucket string
+
+	// Name of the object.
+	Object string
+
+	// Upload ID identifying the multipart upload whose parts are being listed.
+	UploadID string
+
+	// The class of storage used to store the object.
+	StorageClass string
+
+	// Part number after which listing begins.
+	PartNumberMarker int
+
+	// When a list is truncated, this element specifies the last part in the list,
+	// as well as the value to use for the part-number-marker request parameter
+	// in a subsequent request.
+	NextPartNumberMarker int
+
+	// Maximum number of parts that were allowed in the response.
+	MaxParts int
+
+	// Indicates whether the returned list of parts is truncated.
+	IsTruncated bool
+
+	// List of all parts.
+	Parts []partInfo
+
+	EncodingType string // Not supported yet.
+}
+
+// ListMultipartsInfo - represnets bucket resources for incomplete multipart uploads.
+type ListMultipartsInfo struct {
+	// Together with upload-id-marker, this parameter specifies the multipart upload
+	// after which listing should begin.
+	KeyMarker string
+
+	// Together with key-marker, specifies the multipart upload after which listing
+	// should begin. If key-marker is not specified, the upload-id-marker parameter
+	// is ignored.
+	UploadIDMarker string
+
+	// When a list is truncated, this element specifies the value that should be
+	// used for the key-marker request parameter in a subsequent request.
+	NextKeyMarker string
+
+	// When a list is truncated, this element specifies the value that should be
+	// used for the upload-id-marker request parameter in a subsequent request.
+	NextUploadIDMarker string
+
+	// Maximum number of multipart uploads that could have been included in the
+	// response.
+	MaxUploads int
+
+	// Indicates whether the returned list of multipart uploads is truncated. A
+	// value of true indicates that the list was truncated. The list can be truncated
+	// if the number of multipart uploads exceeds the limit allowed or specified
+	// by max uploads.
+	IsTruncated bool
+
+	// List of all pending uploads.
+	Uploads []uploadMetadata
+
+	// When a prefix is provided in the request, The result contains only keys
+	// starting with the specified prefix.
+	Prefix string
+
+	// A character used to truncate the object prefixes.
+	// NOTE: only supported delimiter is '/'.
+	Delimiter string
+
+	// CommonPrefixes contains all (if there are any) keys between Prefix and the
+	// next occurrence of the string specified by delimiter.
+	CommonPrefixes []string
+
+	EncodingType string // Not supported yet.
+}
+
+// ListObjectsInfo - container for list objects.
+type ListObjectsInfo struct {
+	// Indicates whether the returned list objects response is truncated. A
+	// value of true indicates that the list was truncated. The list can be truncated
+	// if the number of objects exceeds the limit allowed or specified
+	// by max keys.
+	IsTruncated bool
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects.
+	//
+	// NOTE: This element is returned only if you have delimiter request parameter
+	// specified.
+	NextMarker string
+
+	// List of objects info for this request.
+	Objects []ObjectInfo
+
+	// List of prefixes for this request.
+	Prefixes []string
+}
+
+// partInfo - represents individual part metadata.
+type partInfo struct {
+	// Part number that identifies the part. This is a positive integer between
+	// 1 and 10,000.
+	PartNumber int
+
+	// Date and time at which the part was uploaded.
+	LastModified time.Time
+
+	// Entity tag returned when the part was initially uploaded.
+	ETag string
+
+	// Size in bytes of the part.
+	Size int64
+}
+
+// uploadMetadata - represents metadata in progress multipart upload.
+type uploadMetadata struct {
+	// Object name for which the multipart upload was initiated.
+	Object string
+
+	// Unique identifier for this multipart upload.
+	UploadID string
+
+	// Date and time at which the multipart upload was initiated.
+	Initiated time.Time
+
+	StorageClass string // Not supported yet.
+}
+
+// completePart - completed part container.
+type completePart struct {
+	// Part number identifying the part. This is a positive integer between 1 and
+	// 10,000
+	PartNumber int
+
+	// Entity tag returned when the part was uploaded.
+	ETag string
+}
+
+// completedParts - is a collection satisfying sort.Interface.
+type completedParts []completePart
+
+func (a completedParts) Len() int           { return len(a) }
+func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
+
+// completeMultipartUpload - represents input fields for completing multipart upload.
+type completeMultipartUpload struct {
+	Parts []completePart `xml:"Part"`
+}

cmd/object-errors.go

@@ -0,0 +1,266 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"io"
+)
+
+// Converts underlying storage error. Convenience function written to
+// handle all cases where we have known types of errors returned by
+// underlying storage layer.
+func toObjectErr(err error, params ...string) error {
+	switch err {
+	case errVolumeNotFound:
+		if len(params) >= 1 {
+			return BucketNotFound{Bucket: params[0]}
+		}
+	case errVolumeNotEmpty:
+		if len(params) >= 1 {
+			return BucketNotEmpty{Bucket: params[0]}
+		}
+	case errVolumeExists:
+		if len(params) >= 1 {
+			return BucketExists{Bucket: params[0]}
+		}
+	case errDiskFull:
+		return StorageFull{}
+	case errIsNotRegular, errFileAccessDenied:
+		if len(params) >= 2 {
+			return ObjectExistsAsDirectory{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		}
+	case errFileNotFound:
+		if len(params) >= 2 {
+			return ObjectNotFound{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		}
+	case errFileNameTooLong:
+		if len(params) >= 2 {
+			return ObjectNameInvalid{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		}
+	case errXLReadQuorum:
+		return InsufficientReadQuorum{}
+	case errXLWriteQuorum:
+		return InsufficientWriteQuorum{}
+	case io.ErrUnexpectedEOF, io.ErrShortWrite:
+		return IncompleteBody{}
+	}
+	return err
+}
+
+// StorageFull storage ran out of space.
+type StorageFull struct{}
+
+func (e StorageFull) Error() string {
+	return "Storage reached its minimum free disk threshold."
+}
+
+// InsufficientReadQuorum storage cannot satisfy quorum for read operation.
+type InsufficientReadQuorum struct{}
+
+func (e InsufficientReadQuorum) Error() string {
+	return "Storage resources are insufficient for the read operation."
+}
+
+// InsufficientWriteQuorum storage cannot satisfy quorum for write operation.
+type InsufficientWriteQuorum struct{}
+
+func (e InsufficientWriteQuorum) Error() string {
+	return "Storage resources are insufficient for the write operation."
+}
+
+// GenericError - generic object layer error.
+type GenericError struct {
+	Bucket string
+	Object string
+}
+
+// BucketNotFound bucket does not exist.
+type BucketNotFound GenericError
+
+func (e BucketNotFound) Error() string {
+	return "Bucket not found: " + e.Bucket
+}
+
+// BucketNotEmpty bucket is not empty.
+type BucketNotEmpty GenericError
+
+func (e BucketNotEmpty) Error() string {
+	return "Bucket not empty: " + e.Bucket
+}
+
+// ObjectNotFound object does not exist.
+type ObjectNotFound GenericError
+
+func (e ObjectNotFound) Error() string {
+	return "Object not found: " + e.Bucket + "#" + e.Object
+}
+
+// ObjectExistsAsDirectory object already exists as a directory.
+type ObjectExistsAsDirectory GenericError
+
+func (e ObjectExistsAsDirectory) Error() string {
+	return "Object exists on : " + e.Bucket + " as directory " + e.Object
+}
+
+// BucketExists bucket exists.
+type BucketExists GenericError
+
+func (e BucketExists) Error() string {
+	return "Bucket exists: " + e.Bucket
+}
+
+// BadDigest - Content-MD5 you specified did not match what we received.
+type BadDigest struct {
+	ExpectedMD5   string
+	CalculatedMD5 string
+}
+
+func (e BadDigest) Error() string {
+	return "Bad digest: Expected " + e.ExpectedMD5 + " is not valid with what we calculated " + e.CalculatedMD5
+}
+
+// UnsupportedDelimiter - unsupported delimiter.
+type UnsupportedDelimiter struct {
+	Delimiter string
+}
+
+func (e UnsupportedDelimiter) Error() string {
+	return fmt.Sprintf("delimiter '%s' is not supported. Only '/' is supported", e.Delimiter)
+}
+
+// InvalidUploadIDKeyCombination - invalid upload id and key marker combination.
+type InvalidUploadIDKeyCombination struct {
+	UploadIDMarker, KeyMarker string
+}
+
+func (e InvalidUploadIDKeyCombination) Error() string {
+	return fmt.Sprintf("Invalid combination of uploadID marker '%s' and marker '%s'", e.UploadIDMarker, e.KeyMarker)
+}
+
+// InvalidMarkerPrefixCombination - invalid marker and prefix combination.
+type InvalidMarkerPrefixCombination struct {
+	Marker, Prefix string
+}
+
+func (e InvalidMarkerPrefixCombination) Error() string {
+	return fmt.Sprintf("Invalid combination of marker '%s' and prefix '%s'", e.Marker, e.Prefix)
+}
+
+// BucketPolicyNotFound - no bucket policy found.
+type BucketPolicyNotFound GenericError
+
+func (e BucketPolicyNotFound) Error() string {
+	return "No bucket policy found for bucket: " + e.Bucket
+}
+
+/// Bucket related errors.
+
+// BucketNameInvalid - bucketname provided is invalid.
+type BucketNameInvalid GenericError
+
+// Return string an error formatted as the given text.
+func (e BucketNameInvalid) Error() string {
+	return "Bucket name invalid: " + e.Bucket
+}
+
+/// Object related errors.
+
+// ObjectNameInvalid - object name provided is invalid.
+type ObjectNameInvalid GenericError
+
+// Return string an error formatted as the given text.
+func (e ObjectNameInvalid) Error() string {
+	return "Object name invalid: " + e.Bucket + "#" + e.Object
+}
+
+// IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header.
+type IncompleteBody GenericError
+
+// Return string an error formatted as the given text.
+func (e IncompleteBody) Error() string {
+	return e.Bucket + "#" + e.Object + "has incomplete body"
+}
+
+// errInvalidRange - returned when given range value is not valid.
+var errInvalidRange = errors.New("Invalid range")
+
+// InvalidRange - invalid range typed error.
+type InvalidRange struct {
+	offsetBegin  int64
+	offsetEnd    int64
+	resourceSize int64
+}
+
+func (e InvalidRange) Error() string {
+	return fmt.Sprintf("The requested range \"bytes %d-%d/%d\" is not satisfiable.", e.offsetBegin, e.offsetEnd, e.resourceSize)
+}
+
+/// Multipart related errors.
+
+// MalformedUploadID malformed upload id.
+type MalformedUploadID struct {
+	UploadID string
+}
+
+func (e MalformedUploadID) Error() string {
+	return "Malformed upload id " + e.UploadID
+}
+
+// InvalidUploadID invalid upload id.
+type InvalidUploadID struct {
+	UploadID string
+}
+
+func (e InvalidUploadID) Error() string {
+	return "Invalid upload id " + e.UploadID
+}
+
+// InvalidPart One or more of the specified parts could not be found
+type InvalidPart struct{}
+
+func (e InvalidPart) Error() string {
+	return "One or more of the specified parts could not be found"
+}
+
+// PartTooSmall - error if part size is less than 5MB.
+type PartTooSmall struct {
+	PartSize   int64
+	PartNumber int
+	PartETag   string
+}
+
+func (e PartTooSmall) Error() string {
+	return fmt.Sprintf("Part size for %d should be atleast 5MB", e.PartNumber)
+}
+
+// NotImplemented If a feature is not implemented
+type NotImplemented struct{}
+
+func (e NotImplemented) Error() string {
+	return "Not Implemented"
+}

cmd/object-handlers-common.go

@@ -0,0 +1,212 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Validates the preconditions for CopyObject, returns true if CopyObject operation should not proceed.
+// Preconditions supported are:
+//  x-amz-copy-source-if-modified-since
+//  x-amz-copy-source-if-unmodified-since
+//  x-amz-copy-source-if-match
+//  x-amz-copy-source-if-none-match
+func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
+	// Return false for methods other than GET and HEAD.
+	if r.Method != "PUT" {
+		return false
+	}
+	// If the object doesn't have a modtime (IsZero), or the modtime
+	// is obviously garbage (Unix time == 0), then ignore modtimes
+	// and don't process the If-Modified-Since header.
+	if objInfo.ModTime.IsZero() || objInfo.ModTime.Equal(time.Unix(0, 0)) {
+		return false
+	}
+
+	// Headers to be set of object content is not going to be written to the client.
+	writeHeaders := func() {
+		// set common headers
+		setCommonHeaders(w)
+
+		// set object-related metadata headers
+		w.Header().Set("Last-Modified", objInfo.ModTime.UTC().Format(http.TimeFormat))
+
+		if objInfo.MD5Sum != "" {
+			w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+		}
+	}
+	// x-amz-copy-source-if-modified-since: Return the object only if it has been modified
+	// since the specified time otherwise return 412 (precondition failed).
+	ifModifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-modified-since")
+	if ifModifiedSinceHeader != "" {
+		if !ifModifiedSince(objInfo.ModTime, ifModifiedSinceHeader) {
+			// If the object is not modified since the specified time.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// x-amz-copy-source-if-unmodified-since : Return the object only if it has not been
+	// modified since the specified time, otherwise return a 412 (precondition failed).
+	ifUnmodifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-unmodified-since")
+	if ifUnmodifiedSinceHeader != "" {
+		if ifModifiedSince(objInfo.ModTime, ifUnmodifiedSinceHeader) {
+			// If the object is modified since the specified time.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// x-amz-copy-source-if-match : Return the object only if its entity tag (ETag) is the
+	// same as the one specified; otherwise return a 412 (precondition failed).
+	ifMatchETagHeader := r.Header.Get("x-amz-copy-source-if-match")
+	if ifMatchETagHeader != "" {
+		if !isETagEqual(objInfo.MD5Sum, ifMatchETagHeader) {
+			// If the object ETag does not match with the specified ETag.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// If-None-Match : Return the object only if its entity tag (ETag) is different from the
+	// one specified otherwise, return a 304 (not modified).
+	ifNoneMatchETagHeader := r.Header.Get("x-amz-copy-source-if-none-match")
+	if ifNoneMatchETagHeader != "" {
+		if isETagEqual(objInfo.MD5Sum, ifNoneMatchETagHeader) {
+			// If the object ETag matches with the specified ETag.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+	// Object content should be written to http.ResponseWriter
+	return false
+}
+
+// Validates the preconditions. Returns true if GET/HEAD operation should not proceed.
+// Preconditions supported are:
+//  If-Modified-Since
+//  If-Unmodified-Since
+//  If-Match
+//  If-None-Match
+func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
+	// Return false for methods other than GET and HEAD.
+	if r.Method != "GET" && r.Method != "HEAD" {
+		return false
+	}
+	// If the object doesn't have a modtime (IsZero), or the modtime
+	// is obviously garbage (Unix time == 0), then ignore modtimes
+	// and don't process the If-Modified-Since header.
+	if objInfo.ModTime.IsZero() || objInfo.ModTime.Equal(time.Unix(0, 0)) {
+		return false
+	}
+
+	// Headers to be set of object content is not going to be written to the client.
+	writeHeaders := func() {
+		// set common headers
+		setCommonHeaders(w)
+
+		// set object-related metadata headers
+		w.Header().Set("Last-Modified", objInfo.ModTime.UTC().Format(http.TimeFormat))
+
+		if objInfo.MD5Sum != "" {
+			w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+		}
+	}
+	// If-Modified-Since : Return the object only if it has been modified since the specified time,
+	// otherwise return a 304 (not modified).
+	ifModifiedSinceHeader := r.Header.Get("If-Modified-Since")
+	if ifModifiedSinceHeader != "" {
+		if !ifModifiedSince(objInfo.ModTime, ifModifiedSinceHeader) {
+			// If the object is not modified since the specified time.
+			writeHeaders()
+			w.WriteHeader(http.StatusNotModified)
+			return true
+		}
+	}
+
+	// If-Unmodified-Since : Return the object only if it has not been modified since the specified
+	// time, otherwise return a 412 (precondition failed).
+	ifUnmodifiedSinceHeader := r.Header.Get("If-Unmodified-Since")
+	if ifUnmodifiedSinceHeader != "" {
+		if ifModifiedSince(objInfo.ModTime, ifUnmodifiedSinceHeader) {
+			// If the object is modified since the specified time.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// If-Match : Return the object only if its entity tag (ETag) is the same as the one specified;
+	// otherwise return a 412 (precondition failed).
+	ifMatchETagHeader := r.Header.Get("If-Match")
+	if ifMatchETagHeader != "" {
+		if !isETagEqual(objInfo.MD5Sum, ifMatchETagHeader) {
+			// If the object ETag does not match with the specified ETag.
+			writeHeaders()
+			writeErrorResponse(w, r, ErrPreconditionFailed, r.URL.Path)
+			return true
+		}
+	}
+
+	// If-None-Match : Return the object only if its entity tag (ETag) is different from the
+	// one specified otherwise, return a 304 (not modified).
+	ifNoneMatchETagHeader := r.Header.Get("If-None-Match")
+	if ifNoneMatchETagHeader != "" {
+		if isETagEqual(objInfo.MD5Sum, ifNoneMatchETagHeader) {
+			// If the object ETag matches with the specified ETag.
+			writeHeaders()
+			w.WriteHeader(http.StatusNotModified)
+			return true
+		}
+	}
+	// Object content should be written to http.ResponseWriter
+	return false
+}
+
+// returns true if object was modified after givenTime.
+func ifModifiedSince(objTime time.Time, givenTimeStr string) bool {
+	givenTime, err := time.Parse(http.TimeFormat, givenTimeStr)
+	if err != nil {
+		return true
+	}
+	// The Date-Modified header truncates sub-second precision, so
+	// use mtime < t+1s instead of mtime <= t to check for unmodified.
+	if objTime.After(givenTime.Add(1 * time.Second)) {
+		return true
+	}
+	return false
+}
+
+// canonicalizeETag returns ETag with leading and trailing double-quotes removed,
+// if any present
+func canonicalizeETag(etag string) string {
+	canonicalETag := strings.TrimPrefix(etag, "\"")
+	return strings.TrimSuffix(canonicalETag, "\"")
+}
+
+// isETagEqual return true if the canonical representations of two ETag strings
+// are equal, false otherwise
+func isETagEqual(left, right string) bool {
+	return canonicalizeETag(left) == canonicalizeETag(right)
+}

cmd/object-handlers.go

@@ -0,0 +1,867 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/hex"
+	"encoding/xml"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+
+	mux "github.com/gorilla/mux"
+)
+
+// supportedGetReqParams - supported request parameters for GET presigned request.
+var supportedGetReqParams = map[string]string{
+	"response-expires":             "Expires",
+	"response-content-type":        "Content-Type",
+	"response-cache-control":       "Cache-Control",
+	"response-content-encoding":    "Content-Encoding",
+	"response-content-language":    "Content-Language",
+	"response-content-disposition": "Content-Disposition",
+}
+
+// setGetRespHeaders - set any requested parameters as response headers.
+func setGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
+	for k, v := range reqParams {
+		if header, ok := supportedGetReqParams[k]; ok {
+			w.Header()[header] = v
+		}
+	}
+}
+
+// errAllowableNotFound - For an anon user, return 404 if have ListBucket, 403 otherwise
+// this is in keeping with the permissions sections of the docs of both:
+//   HEAD Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
+//   GET Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
+func errAllowableObjectNotFound(bucket string, r *http.Request) APIErrorCode {
+	if getRequestAuthType(r) == authTypeAnonymous {
+		//we care about the bucket as a whole, not a particular resource
+		url := *r.URL
+		url.Path = "/" + bucket
+
+		if s3Error := enforceBucketPolicy(bucket, "s3:ListBucket", &url); s3Error != ErrNone {
+			return ErrAccessDenied
+		}
+	}
+	return ErrNoSuchKey
+}
+
+// Simple way to convert a func to io.Writer type.
+type funcToWriter func([]byte) (int, error)
+
+func (f funcToWriter) Write(p []byte) (int, error) {
+	return f(p)
+}
+
+// GetObjectHandler - GET Object
+// ----------
+// This implementation of the GET operation retrieves object. To use GET,
+// you must have READ access to the object.
+func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
+	var object, bucket string
+	vars := mux.Vars(r)
+	bucket = vars["bucket"]
+	object = vars["object"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:GetObject", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+	// Fetch object stat info.
+	objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
+	if err != nil {
+		errorIf(err, "Unable to fetch object info.")
+		apiErr := toAPIErrorCode(err)
+		if apiErr == ErrNoSuchKey {
+			apiErr = errAllowableObjectNotFound(bucket, r)
+		}
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+
+	// Get request range.
+	var hrange *httpRange
+	rangeHeader := r.Header.Get("Range")
+	if rangeHeader != "" {
+		if hrange, err = parseRequestRange(rangeHeader, objInfo.Size); err != nil {
+			// Handle only errInvalidRange
+			// Ignore other parse error and treat it as regular Get request like Amazon S3.
+			if err == errInvalidRange {
+				writeErrorResponse(w, r, ErrInvalidRange, r.URL.Path)
+				return
+			}
+
+			// log the error.
+			errorIf(err, "Invalid request range")
+		}
+
+	}
+
+	// Validate pre-conditions if any.
+	if checkPreconditions(w, r, objInfo) {
+		return
+	}
+
+	// Get the object.
+	startOffset := int64(0)
+	length := objInfo.Size
+	if hrange != nil {
+		startOffset = hrange.offsetBegin
+		length = hrange.getLength()
+	}
+	// Indicates if any data was written to the http.ResponseWriter
+	dataWritten := false
+	// io.Writer type which keeps track if any data was written.
+	writer := funcToWriter(func(p []byte) (int, error) {
+		if !dataWritten {
+			// Set headers on the first write.
+			// Set standard object headers.
+			setObjectHeaders(w, objInfo, hrange)
+
+			// Set any additional requested response headers.
+			setGetRespHeaders(w, r.URL.Query())
+
+			dataWritten = true
+		}
+		return w.Write(p)
+	})
+
+	// Reads the object at startOffset and writes to mw.
+	if err := api.ObjectAPI.GetObject(bucket, object, startOffset, length, writer); err != nil {
+		errorIf(err, "Unable to write to client.")
+		if !dataWritten {
+			// Error response only if no data has been written to client yet. i.e if
+			// partial data has already been written before an error
+			// occurred then no point in setting StatusCode and
+			// sending error XML.
+			apiErr := toAPIErrorCode(err)
+			writeErrorResponse(w, r, apiErr, r.URL.Path)
+		}
+		return
+	}
+	if !dataWritten {
+		// If ObjectAPI.GetObject did not return error and no data has
+		// been written it would mean that it is a 0-byte object.
+		// call wrter.Write(nil) to set appropriate headers.
+		writer.Write(nil)
+	}
+}
+
+// HeadObjectHandler - HEAD Object
+// -----------
+// The HEAD operation retrieves metadata from an object without returning the object itself.
+func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
+	var object, bucket string
+	vars := mux.Vars(r)
+	bucket = vars["bucket"]
+	object = vars["object"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:GetObject", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
+	if err != nil {
+		errorIf(err, "Unable to fetch object info.")
+		apiErr := toAPIErrorCode(err)
+		if apiErr == ErrNoSuchKey {
+			apiErr = errAllowableObjectNotFound(bucket, r)
+		}
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+
+	// Validate pre-conditions if any.
+	if checkPreconditions(w, r, objInfo) {
+		return
+	}
+
+	// Set standard object headers.
+	setObjectHeaders(w, objInfo, nil)
+
+	// Successful response.
+	w.WriteHeader(http.StatusOK)
+}
+
+// CopyObjectHandler - Copy Object
+// ----------
+// This implementation of the PUT operation adds an object to a bucket
+// while reading the object from another source.
+func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:PutObject", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// TODO: Reject requests where body/payload is present, for now we don't even read it.
+
+	// objectSource
+	objectSource, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
+	if err != nil {
+		// Save unescaped string as is.
+		objectSource = r.Header.Get("X-Amz-Copy-Source")
+	}
+
+	// Skip the first element if it is '/', split the rest.
+	if strings.HasPrefix(objectSource, "/") {
+		objectSource = objectSource[1:]
+	}
+	splits := strings.SplitN(objectSource, "/", 2)
+
+	// Save sourceBucket and sourceObject extracted from url Path.
+	var sourceBucket, sourceObject string
+	if len(splits) == 2 {
+		sourceBucket = splits[0]
+		sourceObject = splits[1]
+	}
+	// If source object is empty, reply back error.
+	if sourceObject == "" {
+		writeErrorResponse(w, r, ErrInvalidCopySource, r.URL.Path)
+		return
+	}
+
+	// Source and destination objects cannot be same, reply back error.
+	if sourceObject == object && sourceBucket == bucket {
+		writeErrorResponse(w, r, ErrInvalidCopyDest, r.URL.Path)
+		return
+	}
+
+	objInfo, err := api.ObjectAPI.GetObjectInfo(sourceBucket, sourceObject)
+	if err != nil {
+		errorIf(err, "Unable to fetch object info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), objectSource)
+		return
+	}
+
+	// Verify before x-amz-copy-source preconditions before continuing with CopyObject.
+	if checkCopyObjectPreconditions(w, r, objInfo) {
+		return
+	}
+
+	/// maximum Upload size for object in a single CopyObject operation.
+	if isMaxObjectSize(objInfo.Size) {
+		writeErrorResponse(w, r, ErrEntityTooLarge, objectSource)
+		return
+	}
+
+	pipeReader, pipeWriter := io.Pipe()
+	go func() {
+		startOffset := int64(0) // Read the whole file.
+		// Get the object.
+		gErr := api.ObjectAPI.GetObject(sourceBucket, sourceObject, startOffset, objInfo.Size, pipeWriter)
+		if gErr != nil {
+			errorIf(gErr, "Unable to read an object.")
+			pipeWriter.CloseWithError(gErr)
+			return
+		}
+		pipeWriter.Close() // Close.
+	}()
+
+	// Size of object.
+	size := objInfo.Size
+
+	// Save metadata.
+	metadata := make(map[string]string)
+	// Save other metadata if available.
+	metadata = objInfo.UserDefined
+
+	// Do not set `md5sum` as CopyObject will not keep the
+	// same md5sum as the source.
+
+	// Create the object.
+	md5Sum, err := api.ObjectAPI.PutObject(bucket, object, size, pipeReader, metadata)
+	if err != nil {
+		errorIf(err, "Unable to create an object.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	objInfo, err = api.ObjectAPI.GetObjectInfo(bucket, object)
+	if err != nil {
+		errorIf(err, "Unable to fetch object info.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	response := generateCopyObjectResponse(md5Sum, objInfo.ModTime)
+	encodedSuccessResponse := encodeResponse(response)
+	// write headers
+	setCommonHeaders(w)
+	// write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+	// Explicitly close the reader, to avoid fd leaks.
+	pipeReader.Close()
+
+	if eventN.IsBucketNotificationSet(bucket) {
+		// Notify object created event.
+		eventNotify(eventData{
+			Type:    ObjectCreatedCopy,
+			Bucket:  bucket,
+			ObjInfo: objInfo,
+			ReqParams: map[string]string{
+				"sourceIPAddress": r.RemoteAddr,
+			},
+		})
+	}
+}
+
+// PutObjectHandler - PUT Object
+// ----------
+// This implementation of the PUT operation adds an object to a bucket.
+func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
+	// If the matching failed, it means that the X-Amz-Copy-Source was
+	// wrong, fail right here.
+	if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
+		writeErrorResponse(w, r, ErrInvalidCopySource, r.URL.Path)
+		return
+	}
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	// Get Content-Md5 sent by client and verify if valid
+	md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5"))
+	if err != nil {
+		errorIf(err, "Unable to validate content-md5 format.")
+		writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path)
+		return
+	}
+	/// if Content-Length is unknown/missing, deny the request
+	size := r.ContentLength
+	rAuthType := getRequestAuthType(r)
+	if rAuthType == authTypeStreamingSigned {
+		sizeStr := r.Header.Get("x-amz-decoded-content-length")
+		size, err = strconv.ParseInt(sizeStr, 10, 64)
+		if err != nil {
+			errorIf(err, "Unable to parse `x-amz-decoded-content-length` into its integer value", sizeStr)
+			writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+			return
+		}
+	}
+	if size == -1 && !contains(r.TransferEncoding, "chunked") {
+		writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+		return
+	}
+	/// maximum Upload size for objects in a single operation
+	if isMaxObjectSize(size) {
+		writeErrorResponse(w, r, ErrEntityTooLarge, r.URL.Path)
+		return
+	}
+
+	// Extract metadata to be saved from incoming HTTP header.
+	metadata := extractMetadataFromHeader(r.Header)
+	// Make sure we hex encode md5sum here.
+	metadata["md5Sum"] = hex.EncodeToString(md5Bytes)
+
+	var md5Sum string
+	switch rAuthType {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:PutObject", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+		// Create anonymous object.
+		md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, metadata)
+	case authTypeStreamingSigned:
+		// Initialize stream signature verifier.
+		reader, s3Error := newSignV4ChunkedReader(r)
+		if s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+		md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata)
+	case authTypePresigned, authTypeSigned:
+		// Initialize signature verifier.
+		reader := newSignVerify(r)
+		// Create object.
+		md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata)
+	}
+	if err != nil {
+		errorIf(err, "Unable to create an object.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	if md5Sum != "" {
+		w.Header().Set("ETag", "\""+md5Sum+"\"")
+	}
+	writeSuccessResponse(w, nil)
+
+	// Fetch object info for notifications.
+	objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
+	if err != nil {
+		errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
+		return
+	}
+
+	if eventN.IsBucketNotificationSet(bucket) {
+		// Notify object created event.
+		eventNotify(eventData{
+			Type:    ObjectCreatedPut,
+			Bucket:  bucket,
+			ObjInfo: objInfo,
+			ReqParams: map[string]string{
+				"sourceIPAddress": r.RemoteAddr,
+			},
+		})
+	}
+}
+
+/// Multipart objectAPIHandlers
+
+// NewMultipartUploadHandler - New multipart upload
+func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
+	var object, bucket string
+	vars := mux.Vars(r)
+	bucket = vars["bucket"]
+	object = vars["object"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:PutObject", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// Extract metadata that needs to be saved.
+	metadata := extractMetadataFromHeader(r.Header)
+
+	uploadID, err := api.ObjectAPI.NewMultipartUpload(bucket, object, metadata)
+	if err != nil {
+		errorIf(err, "Unable to initiate new multipart upload id.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+
+	response := generateInitiateMultipartUploadResponse(bucket, object, uploadID)
+	encodedSuccessResponse := encodeResponse(response)
+	// write headers
+	setCommonHeaders(w)
+	// write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// PutObjectPartHandler - Upload part
+func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	// get Content-Md5 sent by client and verify if valid
+	md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5"))
+	if err != nil {
+		writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path)
+		return
+	}
+
+	/// if Content-Length is unknown/missing, throw away
+	size := r.ContentLength
+
+	rAuthType := getRequestAuthType(r)
+	// For auth type streaming signature, we need to gather a different content length.
+	if rAuthType == authTypeStreamingSigned {
+		sizeStr := r.Header.Get("x-amz-decoded-content-length")
+		size, err = strconv.ParseInt(sizeStr, 10, 64)
+		if err != nil {
+			errorIf(err, "Unable to parse `x-amz-decoded-content-length` into its integer value", sizeStr)
+			writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+			return
+		}
+	}
+	if size == -1 {
+		writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+		return
+	}
+
+	/// maximum Upload size for multipart objects in a single operation
+	if isMaxObjectSize(size) {
+		writeErrorResponse(w, r, ErrEntityTooLarge, r.URL.Path)
+		return
+	}
+
+	uploadID := r.URL.Query().Get("uploadId")
+	partIDString := r.URL.Query().Get("partNumber")
+
+	partID, err := strconv.Atoi(partIDString)
+	if err != nil {
+		writeErrorResponse(w, r, ErrInvalidPart, r.URL.Path)
+		return
+	}
+
+	// check partID with maximum part ID for multipart objects
+	if isMaxPartID(partID) {
+		writeErrorResponse(w, r, ErrInvalidMaxParts, r.URL.Path)
+		return
+	}
+
+	var partMD5 string
+	incomingMD5 := hex.EncodeToString(md5Bytes)
+	switch rAuthType {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:PutObject", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+		// No need to verify signature, anonymous request access is already allowed.
+		partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, r.Body, incomingMD5)
+	case authTypeStreamingSigned:
+		// Initialize stream signature verifier.
+		reader, s3Error := newSignV4ChunkedReader(r)
+		if s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+		partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5)
+	case authTypePresigned, authTypeSigned:
+		// Initialize signature verifier.
+		reader := newSignVerify(r)
+		partMD5, err = api.ObjectAPI.PutObjectPart(bucket, object, uploadID, partID, size, reader, incomingMD5)
+	}
+	if err != nil {
+		errorIf(err, "Unable to create object part.")
+		// Verify if the underlying error is signature mismatch.
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	if partMD5 != "" {
+		w.Header().Set("ETag", "\""+partMD5+"\"")
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// AbortMultipartUploadHandler - Abort multipart upload
+func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:AbortMultipartUpload", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	uploadID, _, _, _ := getObjectResources(r.URL.Query())
+	if err := api.ObjectAPI.AbortMultipartUpload(bucket, object, uploadID); err != nil {
+		errorIf(err, "Unable to abort multipart upload.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	writeSuccessNoContent(w)
+}
+
+// ListObjectPartsHandler - List object parts
+func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:ListMultipartUploadParts", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
+	if partNumberMarker < 0 {
+		writeErrorResponse(w, r, ErrInvalidPartNumberMarker, r.URL.Path)
+		return
+	}
+	if maxParts < 0 {
+		writeErrorResponse(w, r, ErrInvalidMaxParts, r.URL.Path)
+		return
+	}
+	listPartsInfo, err := api.ObjectAPI.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
+	if err != nil {
+		errorIf(err, "Unable to list uploaded parts.")
+		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
+		return
+	}
+	response := generateListPartsResponse(listPartsInfo)
+	encodedSuccessResponse := encodeResponse(response)
+	// Write headers.
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// CompleteMultipartUploadHandler - Complete multipart upload
+func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	// Get upload id.
+	uploadID, _, _, _ := getObjectResources(r.URL.Query())
+
+	var md5Sum string
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:PutObject", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+	completeMultipartBytes, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		errorIf(err, "Unable to complete multipart upload.")
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
+	complMultipartUpload := &completeMultipartUpload{}
+	if err = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); err != nil {
+		errorIf(err, "Unable to parse complete multipart upload XML.")
+		writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
+		return
+	}
+	if len(complMultipartUpload.Parts) == 0 {
+		writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
+		return
+	}
+	if !sort.IsSorted(completedParts(complMultipartUpload.Parts)) {
+		writeErrorResponse(w, r, ErrInvalidPartOrder, r.URL.Path)
+		return
+	}
+	// Complete parts.
+	var completeParts []completePart
+	for _, part := range complMultipartUpload.Parts {
+		part.ETag = strings.TrimPrefix(part.ETag, "\"")
+		part.ETag = strings.TrimSuffix(part.ETag, "\"")
+		completeParts = append(completeParts, part)
+	}
+	// Complete multipart upload.
+	// Send 200 OK
+	setCommonHeaders(w)
+	w.WriteHeader(http.StatusOK)
+	// Xml headers need to be sent before we possibly send whitespace characters
+	// to the client.
+	_, err = w.Write([]byte(xml.Header))
+	if err != nil {
+		errorIf(err, "Unable to write XML header for complete multipart upload")
+		writeErrorResponseNoHeader(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
+
+	doneCh := make(chan struct{})
+	// Signal that completeMultipartUpload is over via doneCh
+	go func(doneCh chan<- struct{}) {
+		md5Sum, err = api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts)
+		doneCh <- struct{}{}
+	}(doneCh)
+
+	sendWhiteSpaceChars(w, doneCh)
+	if err != nil {
+		errorIf(err, "Unable to complete multipart upload.")
+		switch oErr := err.(type) {
+		case PartTooSmall:
+			// Write part too small error.
+			writePartSmallErrorResponse(w, r, oErr)
+		default:
+			// Handle all other generic issues.
+			writeErrorResponseNoHeader(w, r, toAPIErrorCode(err), r.URL.Path)
+		}
+		return
+	}
+
+	// Get object location.
+	location := getLocation(r)
+	// Generate complete multipart response.
+	response := generateCompleteMultpartUploadResponse(bucket, object, location, md5Sum)
+	encodedSuccessResponse, err := xml.Marshal(response)
+	if err != nil {
+		errorIf(err, "Unable to parse CompleteMultipartUpload response")
+		writeErrorResponseNoHeader(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
+
+	// Write success response.
+	w.Write(encodedSuccessResponse)
+	w.(http.Flusher).Flush()
+
+	// Fetch object info for notifications.
+	objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
+	if err != nil {
+		errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
+		return
+	}
+
+	if eventN.IsBucketNotificationSet(bucket) {
+		// Notify object created event.
+		eventNotify(eventData{
+			Type:    ObjectCreatedCompleteMultipartUpload,
+			Bucket:  bucket,
+			ObjInfo: objInfo,
+			ReqParams: map[string]string{
+				"sourceIPAddress": r.RemoteAddr,
+			},
+		})
+	}
+}
+
+/// Delete objectAPIHandlers
+
+// DeleteObjectHandler - delete an object
+func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy(bucket, "s3:DeleteObject", r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypeSigned, authTypePresigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+	/// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
+	/// Ignore delete object errors, since we are suppposed to reply
+	/// only 204.
+	if err := api.ObjectAPI.DeleteObject(bucket, object); err != nil {
+		writeSuccessNoContent(w)
+		return
+	}
+	writeSuccessNoContent(w)
+
+	if eventN.IsBucketNotificationSet(bucket) {
+		// Notify object deleted event.
+		eventNotify(eventData{
+			Type:   ObjectRemovedDelete,
+			Bucket: bucket,
+			ObjInfo: ObjectInfo{
+				Name: object,
+			},
+			ReqParams: map[string]string{
+				"sourceIPAddress": r.RemoteAddr,
+			},
+		})
+	}
+}

cmd/object-handlers_test.go

@@ -0,0 +1,334 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+)
+
+// Wrapper for calling GetObject API handler tests for both XL multiple disks and FS single drive setup.
+func TestAPIGetOjectHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testAPIGetOjectHandler)
+}
+
+func testAPIGetOjectHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+	objectName := "test-object"
+	// Create bucket.
+	err := obj.MakeBucket(bucketName)
+	if err != nil {
+		// failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+	// Register the API end points with XL/FS object layer.
+	// Registering only the GetObject handler.
+	apiRouter := initTestAPIEndPoints(obj, []string{"GetObject"})
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	credentials := serverConfig.GetCredential()
+
+	// set of byte data for PutObject.
+	// object has to be inserted before running tests for GetObject.
+	// this is required even to assert the GetObject data,
+	// since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical.
+	bytesData := []struct {
+		byteData []byte
+	}{
+		{generateBytesData(6 * 1024 * 1024)},
+	}
+	// set of inputs for uploading the objects before tests for downloading is done.
+	putObjectInputs := []struct {
+		bucketName    string
+		objectName    string
+		contentLength int64
+		textData      []byte
+		metaData      map[string]string
+	}{
+		// case - 1.
+		{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
+	}
+	// iterate through the above set of inputs and upload the object.
+	for i, input := range putObjectInputs {
+		// uploading the object.
+		_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
+		// if object upload fails stop the test.
+		if err != nil {
+			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
+		}
+	}
+
+	// test cases with inputs and expected result for GetObject.
+	testCases := []struct {
+		bucketName string
+		objectName string
+		byteRange  string // range of bytes to be fetched from GetObject.
+		// expected output.
+		expectedContent    []byte // expected response body.
+		expectedRespStatus int    // expected response status body.
+	}{
+		// Test case - 1.
+		// Fetching the entire object and validating its contents.
+		{
+			bucketName:         bucketName,
+			objectName:         objectName,
+			byteRange:          "",
+			expectedContent:    bytesData[0].byteData,
+			expectedRespStatus: http.StatusOK,
+		},
+		// Test case - 2.
+		// Case with non-existent object name.
+		{
+			bucketName:         bucketName,
+			objectName:         "abcd",
+			byteRange:          "",
+			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey), getGetObjectURL("", bucketName, "abcd"))),
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 3.
+		// Requesting from range 10-100.
+		{
+			bucketName:         bucketName,
+			objectName:         objectName,
+			byteRange:          "bytes=10-100",
+			expectedContent:    bytesData[0].byteData[10:101],
+			expectedRespStatus: http.StatusPartialContent,
+		},
+		// Test case - 4.
+		// Test case with invalid range.
+		{
+			bucketName:         bucketName,
+			objectName:         objectName,
+			byteRange:          "bytes=-0",
+			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidRange), getGetObjectURL("", bucketName, objectName))),
+			expectedRespStatus: http.StatusRequestedRangeNotSatisfiable,
+		},
+		// Test case - 5.
+		// Test case with byte range exceeding the object size.
+		// Expected to read till end of the object.
+		{
+			bucketName:         bucketName,
+			objectName:         objectName,
+			byteRange:          "bytes=10-1000000000000000",
+			expectedContent:    bytesData[0].byteData[10:],
+			expectedRespStatus: http.StatusPartialContent,
+		},
+	}
+	// Iterating over the cases, fetching the object validating the response.
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for Get Object end point.
+		req, err := newTestSignedRequest("GET", getGetObjectURL("", testCase.bucketName, testCase.objectName),
+			0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for Get Object: <ERROR> %v", i+1, err)
+		}
+		if testCase.byteRange != "" {
+			req.Header.Add("Range", testCase.byteRange)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler`  handles the request.
+		apiRouter.ServeHTTP(rec, req)
+		// Assert the response code with the expected status.
+		if rec.Code != testCase.expectedRespStatus {
+			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code)
+		}
+		// read the response body.
+		actualContent, err := ioutil.ReadAll(rec.Body)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Verify whether the bucket obtained object is same as the one inserted.
+		if !bytes.Equal(testCase.expectedContent, actualContent) {
+			t.Errorf("Test %d: %s: Object content differs from expected value.: %s", i+1, instanceType, string(actualContent))
+		}
+	}
+}
+
+// Wrapper for calling Copy Object API handler tests for both XL multiple disks and single node setup.
+func TestAPICopyObjectHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testAPICopyObjectHandler)
+}
+
+func testAPICopyObjectHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// get random bucket name.
+	bucketName := getRandomBucketName()
+	objectName := "test-object"
+	// Create bucket.
+	err := obj.MakeBucket(bucketName)
+	if err != nil {
+		// failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err)
+	}
+	// Register the API end points with XL/FS object layer.
+	// Registering only the Copy Object handler.
+	apiRouter := initTestAPIEndPoints(obj, []string{"CopyObject"})
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	err = initEventNotifier(obj)
+	if err != nil {
+		t.Fatalf("Initializing event notifiers failed")
+	}
+
+	credentials := serverConfig.GetCredential()
+
+	// set of byte data for PutObject.
+	// object has to be inserted before running tests for Copy Object.
+	// this is required even to assert the copied object,
+	bytesData := []struct {
+		byteData []byte
+	}{
+		{generateBytesData(6 * 1024 * 1024)},
+	}
+
+	buffers := []*bytes.Buffer{
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+	}
+
+	// set of inputs for uploading the objects before tests for downloading is done.
+	putObjectInputs := []struct {
+		bucketName    string
+		objectName    string
+		contentLength int64
+		textData      []byte
+		metaData      map[string]string
+	}{
+		// case - 1.
+		{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
+	}
+	// iterate through the above set of inputs and upload the object.
+	for i, input := range putObjectInputs {
+		// uploading the object.
+		_, err = obj.PutObject(input.bucketName, input.objectName, input.contentLength, bytes.NewBuffer(input.textData), input.metaData)
+		// if object upload fails stop the test.
+		if err != nil {
+			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
+		}
+	}
+
+	// test cases with inputs and expected result for Copy Object.
+	testCases := []struct {
+		bucketName       string
+		newObjectName    string // name of the newly copied object.
+		copySourceHeader string // data for "X-Amz-Copy-Source" header. Contains the object to be copied in the URL.
+		// expected output.
+		expectedRespStatus int
+	}{
+		// Test case - 1.
+		{
+			bucketName:         bucketName,
+			newObjectName:      "newObject1",
+			copySourceHeader:   url.QueryEscape("/" + bucketName + "/" + objectName),
+			expectedRespStatus: http.StatusOK,
+		},
+
+		// Test case - 2.
+		// Test case with invalid source object.
+		{
+			bucketName:         bucketName,
+			newObjectName:      "newObject1",
+			copySourceHeader:   url.QueryEscape("/"),
+			expectedRespStatus: http.StatusBadRequest,
+		},
+		// Test case - 3.
+		// Test case with new object name is same as object to be copied.
+		{
+			bucketName:         bucketName,
+			newObjectName:      objectName,
+			copySourceHeader:   url.QueryEscape("/" + bucketName + "/" + objectName),
+			expectedRespStatus: http.StatusBadRequest,
+		},
+		// Test case - 4.
+		// Test case with non-existent source file.
+		// Case for the purpose of failing `api.ObjectAPI.GetObjectInfo`.
+		// Expecting the response status code to http.StatusNotFound (404).
+		{
+			bucketName:         bucketName,
+			newObjectName:      objectName,
+			copySourceHeader:   url.QueryEscape("/" + bucketName + "/" + "non-existent-object"),
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 5.
+		// Test case with non-existent source file.
+		// Case for the purpose of failing `api.ObjectAPI.PutObject`.
+		// Expecting the response status code to http.StatusNotFound (404).
+		{
+			bucketName:         "non-existent-destination-bucket",
+			newObjectName:      objectName,
+			copySourceHeader:   url.QueryEscape("/" + bucketName + "/" + objectName),
+			expectedRespStatus: http.StatusNotFound,
+		},
+	}
+
+	for i, testCase := range testCases {
+		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for copy object.
+		req, err := newTestSignedRequest("PUT", getCopyObjectURL("", testCase.bucketName, testCase.newObjectName),
+			0, nil, credentials.AccessKeyID, credentials.SecretAccessKey)
+
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
+		}
+		// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
+		if testCase.copySourceHeader != "" {
+			req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
+		// Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request.
+		apiRouter.ServeHTTP(rec, req)
+		// Assert the response code with the expected status.
+		if rec.Code != testCase.expectedRespStatus {
+			t.Fatalf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+		}
+		if rec.Code == http.StatusOK {
+			// See if the new object is formed.
+			// testing whether the copy was successful.
+			err = obj.GetObject(testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0])
+			if err != nil {
+				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
+			}
+			if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) {
+				t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i+1, instanceType)
+			}
+			buffers[0].Reset()
+		}
+	}
+}

cmd/object-interface.go

@@ -0,0 +1,49 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "io"
+
+// ObjectLayer implements primitives for object API layer.
+type ObjectLayer interface {
+	// Storage operations.
+	Shutdown() error
+	StorageInfo() StorageInfo
+
+	// Bucket operations.
+	MakeBucket(bucket string) error
+	GetBucketInfo(bucket string) (bucketInfo BucketInfo, err error)
+	ListBuckets() (buckets []BucketInfo, err error)
+	DeleteBucket(bucket string) error
+	ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
+	ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error)
+
+	// Object operations.
+	GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
+	GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
+	PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string) (md5 string, err error)
+	DeleteObject(bucket, object string) error
+	HealObject(bucket, object string) error
+
+	// Multipart operations.
+	ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)
+	NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error)
+	PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (md5 string, err error)
+	ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error)
+	AbortMultipartUpload(bucket, object, uploadID string) error
+	CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (md5 string, err error)
+}

cmd/object-multipart-common.go

@@ -0,0 +1,166 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"path"
+	"sort"
+	"sync"
+	"time"
+)
+
+// A uploadInfo represents the s3 compatible spec.
+type uploadInfo struct {
+	UploadID  string    `json:"uploadId"`  // UploadID for the active multipart upload.
+	Deleted   bool      `json:"deleted"`   // Currently unused, for future use.
+	Initiated time.Time `json:"initiated"` // Indicates when the uploadID was initiated.
+}
+
+// A uploadsV1 represents `uploads.json` metadata header.
+type uploadsV1 struct {
+	Version string       `json:"version"`   // Version of the current `uploads.json`
+	Format  string       `json:"format"`    // Format of the current `uploads.json`
+	Uploads []uploadInfo `json:"uploadIds"` // Captures all the upload ids for a given object.
+}
+
+// byInitiatedTime is a collection satisfying sort.Interface.
+type byInitiatedTime []uploadInfo
+
+func (t byInitiatedTime) Len() int      { return len(t) }
+func (t byInitiatedTime) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
+func (t byInitiatedTime) Less(i, j int) bool {
+	return t[i].Initiated.Before(t[j].Initiated)
+}
+
+// AddUploadID - adds a new upload id in order of its initiated time.
+func (u *uploadsV1) AddUploadID(uploadID string, initiated time.Time) {
+	u.Uploads = append(u.Uploads, uploadInfo{
+		UploadID:  uploadID,
+		Initiated: initiated,
+	})
+	sort.Sort(byInitiatedTime(u.Uploads))
+}
+
+// Index - returns the index of matching the upload id.
+func (u uploadsV1) Index(uploadID string) int {
+	for i, u := range u.Uploads {
+		if u.UploadID == uploadID {
+			return i
+		}
+	}
+	return -1
+}
+
+// readUploadsJSON - get all the saved uploads JSON.
+func readUploadsJSON(bucket, object string, disk StorageAPI) (uploadIDs uploadsV1, err error) {
+	uploadJSONPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
+	// Reads entire `uploads.json`.
+	buf, err := disk.ReadAll(minioMetaBucket, uploadJSONPath)
+	if err != nil {
+		return uploadsV1{}, err
+	}
+
+	// Decode `uploads.json`.
+	if err = json.Unmarshal(buf, &uploadIDs); err != nil {
+		return uploadsV1{}, err
+	}
+
+	// Success.
+	return uploadIDs, nil
+}
+
+// newUploadsV1 - initialize new uploads v1.
+func newUploadsV1(format string) uploadsV1 {
+	uploadIDs := uploadsV1{}
+	uploadIDs.Version = "1.0.0" // Should follow semantic versioning.
+	uploadIDs.Format = format
+	return uploadIDs
+}
+
+// Wrapper which removes all the uploaded parts.
+func cleanupUploadedParts(bucket, object, uploadID string, storageDisks ...StorageAPI) error {
+	var errs = make([]error, len(storageDisks))
+	var wg = &sync.WaitGroup{}
+
+	// Construct uploadIDPath.
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+
+	// Cleanup uploadID for all disks.
+	for index, disk := range storageDisks {
+		if disk == nil {
+			errs[index] = errDiskNotFound
+			continue
+		}
+		wg.Add(1)
+		// Cleanup each uploadID in a routine.
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			err := cleanupDir(disk, minioMetaBucket, uploadIDPath)
+			if err != nil {
+				errs[index] = err
+				return
+			}
+			errs[index] = nil
+		}(index, disk)
+	}
+
+	// Wait for all the cleanups to finish.
+	wg.Wait()
+
+	// Return first error.
+	for _, err := range errs {
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// listMultipartUploadIDs - list all the upload ids from a marker up to 'count'.
+func listMultipartUploadIDs(bucketName, objectName, uploadIDMarker string, count int, disk StorageAPI) ([]uploadMetadata, bool, error) {
+	var uploads []uploadMetadata
+	// Read `uploads.json`.
+	uploadsJSON, err := readUploadsJSON(bucketName, objectName, disk)
+	if err != nil {
+		return nil, false, err
+	}
+	index := 0
+	if uploadIDMarker != "" {
+		for ; index < len(uploadsJSON.Uploads); index++ {
+			if uploadsJSON.Uploads[index].UploadID == uploadIDMarker {
+				// Skip the uploadID as it would already be listed in previous listing.
+				index++
+				break
+			}
+		}
+	}
+	for index < len(uploadsJSON.Uploads) {
+		uploads = append(uploads, uploadMetadata{
+			Object:    objectName,
+			UploadID:  uploadsJSON.Uploads[index].UploadID,
+			Initiated: uploadsJSON.Uploads[index].Initiated,
+		})
+		count--
+		index++
+		if count == 0 {
+			break
+		}
+	}
+	end := (index == len(uploadsJSON.Uploads))
+	return uploads, end, nil
+}

cmd/object-utils.go

@@ -0,0 +1,166 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"path"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/skyrings/skyring-common/tools/uuid"
+)
+
+const (
+	// Minio meta bucket.
+	minioMetaBucket = ".minio.sys"
+	// Multipart meta prefix.
+	mpartMetaPrefix = "multipart"
+	// Tmp meta prefix.
+	tmpMetaPrefix = "tmp"
+)
+
+// validBucket regexp.
+var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
+var isIPAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`)
+
+// IsValidBucketName verifies a bucket name in accordance with Amazon's
+// requirements. It must be 3-63 characters long, can contain dashes
+// and periods, but must begin and end with a lowercase letter or a number.
+// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
+func IsValidBucketName(bucket string) bool {
+	// Special case when bucket is equal to 'metaBucket'.
+	if bucket == minioMetaBucket {
+		return true
+	}
+	if len(bucket) < 3 || len(bucket) > 63 {
+		return false
+	}
+	if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
+		return false
+	}
+	return (validBucket.MatchString(bucket) &&
+		!isIPAddress.MatchString(bucket) &&
+		!strings.Contains(bucket, ".."))
+}
+
+// IsValidObjectName verifies an object name in accordance with Amazon's
+// requirements. It cannot exceed 1024 characters and must be a valid UTF8
+// string.
+//
+// See:
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
+//
+// You should avoid the following characters in a key name because of
+// significant special handling for consistency across all
+// applications.
+//
+// Rejects strings with following characters.
+//
+// - Backslash ("\")
+// - Caret ("^")
+// - Grave accent / back tick ("`")
+// - Vertical bar / pipe ("|")
+//
+// Minio does not support object names with trailing "/".
+func IsValidObjectName(object string) bool {
+	if len(object) == 0 {
+		return false
+	}
+	if strings.HasSuffix(object, slashSeparator) {
+		return false
+	}
+	if strings.HasPrefix(object, slashSeparator) {
+		return false
+	}
+	return IsValidObjectPrefix(object)
+}
+
+// IsValidObjectPrefix verifies whether the prefix is a valid object name.
+// Its valid to have a empty prefix.
+func IsValidObjectPrefix(object string) bool {
+	if len(object) > 1024 {
+		return false
+	}
+	if !utf8.ValidString(object) {
+		return false
+	}
+	// Reject unsupported characters in object name.
+	if strings.ContainsAny(object, "`^|\\\"") {
+		return false
+	}
+	return true
+}
+
+// Slash separator.
+const slashSeparator = "/"
+
+// retainSlash - retains slash from a path.
+func retainSlash(s string) string {
+	return strings.TrimSuffix(s, slashSeparator) + slashSeparator
+}
+
+// pathJoin - like path.Join() but retains trailing "/" of the last element
+func pathJoin(elem ...string) string {
+	trailingSlash := ""
+	if len(elem) > 0 {
+		if strings.HasSuffix(elem[len(elem)-1], slashSeparator) {
+			trailingSlash = "/"
+		}
+	}
+	return path.Join(elem...) + trailingSlash
+}
+
+// getUUID() - get a unique uuid.
+func getUUID() (uuidStr string) {
+	for {
+		uuid, err := uuid.New()
+		if err != nil {
+			errorIf(err, "Unable to initialize uuid")
+			continue
+		}
+		uuidStr = uuid.String()
+		break
+	}
+	return uuidStr
+}
+
+// Create an s3 compatible MD5sum for complete multipart transaction.
+func completeMultipartMD5(parts ...completePart) (string, error) {
+	var finalMD5Bytes []byte
+	for _, part := range parts {
+		md5Bytes, err := hex.DecodeString(part.ETag)
+		if err != nil {
+			return "", err
+		}
+		finalMD5Bytes = append(finalMD5Bytes, md5Bytes...)
+	}
+	md5Hasher := md5.New()
+	md5Hasher.Write(finalMD5Bytes)
+	s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(parts))
+	return s3MD5, nil
+}
+
+// byBucketName is a collection satisfying sort.Interface.
+type byBucketName []BucketInfo
+
+func (d byBucketName) Len() int           { return len(d) }
+func (d byBucketName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d byBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }

cmd/object-utils_test.go

@@ -0,0 +1,110 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"testing"
+)
+
+// Tests validate bucket name.
+func TestIsValidBucketName(t *testing.T) {
+	testCases := []struct {
+		bucketName string
+		shouldPass bool
+	}{
+		// cases which should pass the test.
+		// passing in valid bucket names.
+		{"lol", true},
+		{"1-this-is-valid", true},
+		{"1-this-too-is-valid-1", true},
+		{"this.works.too.1", true},
+		{"1234567", true},
+		{"123", true},
+		{"s3-eu-west-1.amazonaws.com", true},
+		{"ideas-are-more-powerful-than-guns", true},
+		{"testbucket", true},
+		{"1bucket", true},
+		{"bucket1", true},
+		// cases for which test should fail.
+		// passing invalid bucket names.
+		{"------", false},
+		{"my..bucket", false},
+		{"192.168.1.1", false},
+		{"$this-is-not-valid-too", false},
+		{"contains-$-dollar", false},
+		{"contains-^-carrot", false},
+		{"contains-$-dollar", false},
+		{"contains-$-dollar", false},
+		{"......", false},
+		{"", false},
+		{"a", false},
+		{"ab", false},
+		{".starts-with-a-dot", false},
+		{"ends-with-a-dot.", false},
+		{"ends-with-a-dash-", false},
+		{"-starts-with-a-dash", false},
+		{"THIS-BEINGS-WITH-UPPERCASe", false},
+		{"tHIS-ENDS-WITH-UPPERCASE", false},
+		{"ThisBeginsAndEndsWithUpperCase", false},
+		{"una ñina", false},
+		{"lalalallalallalalalallalallalala-theString-size-is-greater-than-64", false},
+	}
+
+	for i, testCase := range testCases {
+		isValidBucketName := IsValidBucketName(testCase.bucketName)
+		if testCase.shouldPass && !isValidBucketName {
+			t.Errorf("Test case %d: Expected \"%s\" to be a valid bucket name", i+1, testCase.bucketName)
+		}
+		if !testCase.shouldPass && isValidBucketName {
+			t.Errorf("Test case %d: Expected bucket name \"%s\" to be invalid", i+1, testCase.bucketName)
+		}
+	}
+}
+
+// Tests for validate object name.
+func TestIsValidObjectName(t *testing.T) {
+	testCases := []struct {
+		objectName string
+		shouldPass bool
+	}{
+		// cases which should pass the test.
+		// passing in valid object name.
+		{"object", true},
+		{"The Shining Script <v1>.pdf", true},
+		{"Cost Benefit Analysis (2009-2010).pptx", true},
+		{"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", true},
+		{"SHØRT", true},
+		{"f*le", true},
+		{"There are far too many object names, and far too few bucket names!", true},
+		// cases for which test should fail.
+		// passing invalid object names.
+		{"", false},
+		{"a/b/c/", false},
+		{"/a/b/c", false},
+		{string([]byte{0xff, 0xfe, 0xfd}), false},
+	}
+
+	for i, testCase := range testCases {
+		isValidObjectName := IsValidObjectName(testCase.objectName)
+		if testCase.shouldPass && !isValidObjectName {
+			t.Errorf("Test case %d: Expected \"%s\" to be a valid object name", i+1, testCase.objectName)
+		}
+		if !testCase.shouldPass && isValidObjectName {
+			t.Errorf("Test case %d: Expected object name \"%s\" to be invalid", i+1, testCase.objectName)
+		}
+	}
+}