update: Deprecate the usage of update=yes query param. (#2801)

Fixes #2799

Dockerfile

@@ -1,15 +1,19 @@
-FROM golang:1.6
+FROM golang:1.6-alpine
 
-RUN mkdir -p /go/src/app
 WORKDIR /go/src/app
+ENV ALLOW_CONTAINER_ROOT=1
 
 COPY . /go/src/app
-RUN go-wrapper download
-RUN go-wrapper install
-
-ENV ALLOW_CONTAINER_ROOT=1
-RUN mkdir -p /export/docker && cp /go/src/app/docs/Docker.md /export/docker/
+RUN \
+	apk add --no-cache git bash && \
+	go-wrapper download && \
+        go-wrapper install -ldflags "`./buildscripts/docker-flags.sh`" && \
+	mkdir -p /export/docker && \
+	cp /go/src/app/docs/Docker.md /export/docker/ && \
+	rm -rf /go/pkg /go/src && \
+	apk del git bash
 
 EXPOSE 9000
-ENTRYPOINT ["go-wrapper", "run", "server"]
+ENTRYPOINT ["minio", "server"]
+VOLUME ["/export"]
 CMD ["/export"]

README.md

@@ -16,6 +16,15 @@ Minio server is light enough to be bundled with the application stack, similar t
 ||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
 |FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
 
+### Install from Homebrew
+
+Install minio packages using [Homebrew](http://brew.sh/)
+ 
+```sh
+$ brew install minio
+$ minio --help
+```
+
 ### Install from Source
 
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
@@ -23,9 +32,8 @@ Source installation is only intended for developers and advanced users. If you d
 
 ```sh
 
-$ go get -d github.com/minio/minio
-$ cd $GOPATH/src/github.com/minio/minio
-$ make
+$ go get -u github.com/minio/minio
+
 
 ```
 

appveyor.yml

@@ -9,7 +9,7 @@ platform: x64
 
 clone_folder: c:\gopath\src\github.com\minio\minio
 
-# environment variables
+# Environment variables
 environment:
   GOPATH: c:\gopath
   GO15VENDOREXPERIMENT: 1
@@ -19,21 +19,36 @@ install:
   - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
   - go version
   - go env
+  - python --version
 
-# to run your custom scripts instead of automatic MSBuild
+# To run your custom scripts instead of automatic MSBuild
 build_script:
-  - go test -race github.com/minio/minio/cmd...
-  - go test -race github.com/minio/minio/pkg...
-  - go test -coverprofile=coverage.txt -covermode=atomic
+  # Compile
+  - appveyor AddCompilationMessage "Starting Compile"
+  - cd c:\gopath\src\github.com\minio\minio
   - go run buildscripts/gen-ldflags.go > temp.txt
   - set /p BUILD_LDFLAGS=<temp.txt
   - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
+  - appveyor AddCompilationMessage "Compile Success"
+
+# To run your custom scripts instead of automatic tests
+test_script:
+  # Unit tests
+  - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
+  - mkdir build\coverage
+  - go test -race github.com/minio/minio/cmd...
+  - go test -race github.com/minio/minio/pkg...
+  - go test -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
 
 after_test:
-  - bash <(curl -s https://codecov.io/bash)
-
-# to disable automatic tests
-test: off
+  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
+  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
+  - ps: Push-AppveyorArtifact build\coverage\coverage.html
+  # Upload coverage report.
+  - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
+  - pip install codecov
+  - codecov -X gcov -f "build\coverage\coverage.txt"
 
 # to disable deployment
 deploy: off

buildscripts/docker-flags.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Minio Cloud Storage, (C) 2016 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+minio="github.com/minio/minio"
+version="2016-09-11T17:42:18Z"
+release="RELEASE.2016-09-11T17-42-18Z"
+commit="85e2d886bcb005d49f3876d6849a2b5a55e03cd3"
+
+echo "-X ${minio}/cmd.Version=${version} -X ${minio}/cmd.ReleaseTag=${release} -X ${minio}/cmd.CommitID=${commit}"
+

cmd/api-errors.go

@@ -103,6 +103,7 @@ const (
 	ErrNegativeExpires
 	ErrAuthHeaderEmpty
 	ErrExpiredPresignRequest
+	ErrRequestNotReadyYet
 	ErrUnsignedHeaders
 	ErrMissingDateHeader
 	ErrInvalidQuerySignatureAlgo
@@ -119,6 +120,7 @@ const (
 	ErrFilterNamePrefix
 	ErrFilterNameSuffix
 	ErrFilterValueInvalid
+	ErrOverlappingConfigs
 
 	// S3 extended errors.
 	ErrContentSHA256Mismatch
@@ -262,7 +264,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrNoSuchBucketPolicy: {
 		Code:           "NoSuchBucketPolicy",
-		Description:    "The specified bucket does not have a bucket policy.",
+		Description:    "The bucket policy does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrNoSuchKey: {
@@ -327,7 +329,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrBucketNotEmpty: {
 		Code:           "BucketNotEmpty",
-		Description:    "The bucket you tried to delete is not empty.",
+		Description:    "The bucket you tried to delete is not empty",
 		HTTPStatusCode: http.StatusConflict,
 	},
 	ErrAllAccessDisabled: {
@@ -445,7 +447,12 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	ErrExpiredPresignRequest: {
 		Code:           "AccessDenied",
 		Description:    "Request has expired",
-		HTTPStatusCode: http.StatusBadRequest,
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrRequestNotReadyYet: {
+		Code:           "AccessDenied",
+		Description:    "Request is not valid yet",
+		HTTPStatusCode: http.StatusForbidden,
 	},
 	// FIXME: Actual XML error response also contains the header which missed in lsit of signed header parameters.
 	ErrUnsignedHeaders: {
@@ -505,6 +512,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Size of filter rule value cannot exceed 1024 bytes in UTF-8 representation",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrOverlappingConfigs: {
+		Code:           "InvalidArgument",
+		Description:    "Configurations overlap. Configurations on the same bucket cannot share a common event type.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	/// S3 extensions.
 	ErrContentSHA256Mismatch: {

cmd/api-resources.go

@@ -36,7 +36,7 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 }
 
 // Parse bucket url queries for ListObjects V2.
-func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, maxkeys int, encodingType string) {
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
 	prefix = values.Get("prefix")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
@@ -47,6 +47,9 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 	}
 	encodingType = values.Get("encoding-type")
 	token = values.Get("continuation-token")
+	if values.Get("fetch-owner") == "true" {
+		fetchOwner = true
+	}
 	return
 }
 

cmd/api-response.go

@@ -226,16 +226,6 @@ type CompleteMultipartUploadResponse struct {
 	ETag     string
 }
 
-// PostResponse container for completed post upload response
-type PostResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PostResponse" json:"-"`
-
-	Location string
-	Bucket   string
-	Key      string
-	ETag     string
-}
-
 // DeleteError structure.
 type DeleteError struct {
 	Code    string
@@ -337,14 +327,16 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, fetchOwner bool, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
 	var contents []Object
 	var prefixes []CommonPrefix
 	var owner = Owner{}
 	var data = ListObjectsV2Response{}
 
-	owner.ID = "minio"
-	owner.DisplayName = "minio"
+	if fetchOwner {
+		owner.ID = "minio"
+		owner.DisplayName = "minio"
+	}
 
 	for _, object := range resp.Objects {
 		var content = Object{}

cmd/auth-handler.go

@@ -165,22 +165,18 @@ func setAuthHandler(h http.Handler) http.Handler {
 }
 
 // List of all support S3 auth types.
-var supportedS3AuthTypes = []authType{
-	authTypeAnonymous,
-	authTypePresigned,
-	authTypeSigned,
-	authTypePostPolicy,
-	authTypeStreamingSigned,
+var supportedS3AuthTypes = map[authType]struct{}{
+	authTypeAnonymous:       {},
+	authTypePresigned:       {},
+	authTypeSigned:          {},
+	authTypePostPolicy:      {},
+	authTypeStreamingSigned: {},
 }
 
 // Validate if the authType is valid and supported.
 func isSupportedS3AuthType(aType authType) bool {
-	for _, a := range supportedS3AuthTypes {
-		if a == aType {
-			return true
-		}
-	}
-	return false
+	_, ok := supportedS3AuthTypes[aType]
+	return ok
 }
 
 // handler for validating incoming authorization headers.

cmd/bucket-handlers-listobjects.go

@@ -82,7 +82,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		}
 	}
 	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
 
 	// In ListObjectsV2 'continuation-token' is the marker.
 	marker := token
@@ -91,7 +91,8 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		// Then we need to use 'start-after' as marker instead.
 		marker = startAfter
 	}
-	// Validate all the query params before beginning to serve the request.
+	// Validate the query params before beginning to serve the request.
+	// fetch-owner is not validated since it is a boolean
 	if s3Error := listObjectsValidateArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(w, r, s3Error, r.URL.Path)
 		return
@@ -106,7 +107,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, maxKeys, listObjectsInfo)
+	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, fetchOwner, maxKeys, listObjectsInfo)
 	// Write headers
 	setCommonHeaders(w)
 	// Write success response.

cmd/bucket-handlers.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"path"
 	"strings"
+	"sync"
 
 	mux "github.com/gorilla/mux"
 	"github.com/minio/minio-go/pkg/set"
@@ -41,7 +42,7 @@ func enforceBucketPolicy(bucket string, action string, reqURL *url.URL) (s3Error
 	}
 
 	// Construct resource in 'arn:aws:s3:::examplebucket/object' format.
-	resource := AWSResourcePrefix + strings.TrimPrefix(reqURL.Path, "/")
+	resource := AWSResourcePrefix + strings.TrimSuffix(strings.TrimPrefix(reqURL.Path, "/"), "/")
 
 	// Get conditions for policy verification.
 	conditionKeyMap := make(map[string]set.StringSet)
@@ -240,24 +241,47 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var deleteErrors []DeleteError
-	var deletedObjects []ObjectIdentifier
-	// Loop through all the objects and delete them sequentially.
-	for _, object := range deleteObjects.Objects {
-		err := api.ObjectAPI.DeleteObject(bucket, object.ObjectName)
-		if err == nil {
-			deletedObjects = append(deletedObjects, ObjectIdentifier{
-				ObjectName: object.ObjectName,
-			})
-		} else {
-			errorIf(err, "Unable to delete object.")
-			deleteErrors = append(deleteErrors, DeleteError{
-				Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
-				Message: errorCodeResponse[toAPIErrorCode(err)].Description,
-				Key:     object.ObjectName,
-			})
-		}
+	var wg = &sync.WaitGroup{} // Allocate a new wait group.
+	var dErrs = make([]error, len(deleteObjects.Objects))
+
+	// Delete all requested objects in parallel.
+	for index, object := range deleteObjects.Objects {
+		wg.Add(1)
+		go func(i int, obj ObjectIdentifier) {
+			defer wg.Done()
+			dErr := api.ObjectAPI.DeleteObject(bucket, obj.ObjectName)
+			if dErr != nil {
+				dErrs[i] = dErr
+			}
+		}(index, object)
 	}
+	wg.Wait()
+
+	// Collect deleted objects and errors if any.
+	var deletedObjects []ObjectIdentifier
+	var deleteErrors []DeleteError
+	for index, err := range dErrs {
+		object := deleteObjects.Objects[index]
+		// Success deleted objects are collected separately.
+		if err == nil {
+			deletedObjects = append(deletedObjects, object)
+			continue
+		}
+		if _, ok := err.(ObjectNotFound); ok {
+			// If the object is not found it should be
+			// accounted as deleted as per S3 spec.
+			deletedObjects = append(deletedObjects, object)
+			continue
+		}
+		errorIf(err, "Unable to delete object. %s", object.ObjectName)
+		// Error during delete should be collected separately.
+		deleteErrors = append(deleteErrors, DeleteError{
+			Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
+			Message: errorCodeResponse[toAPIErrorCode(err)].Description,
+			Key:     object.ObjectName,
+		})
+	}
+
 	// Generate response
 	response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
 	encodedSuccessResponse := encodeResponse(response)
@@ -265,6 +289,22 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	setCommonHeaders(w)
 	// Write success response.
 	writeSuccessResponse(w, encodedSuccessResponse)
+
+	if globalEventNotifier.IsBucketNotificationSet(bucket) {
+		// Notify deleted event for objects.
+		for _, dobj := range deletedObjects {
+			eventNotify(eventData{
+				Type:   ObjectRemovedDelete,
+				Bucket: bucket,
+				ObjInfo: ObjectInfo{
+					Name: dobj.ObjectName,
+				},
+				ReqParams: map[string]string{
+					"sourceIPAddress": r.RemoteAddr,
+				},
+			})
+		}
+	}
 }
 
 // PutBucketHandler - PUT Bucket
@@ -284,6 +324,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	// requests which do not follow valid region requirements.
 	if s3Error := isValidLocationConstraint(r); s3Error != ErrNone {
 		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
 	}
 
 	// Proceed to creating a bucket.
@@ -352,25 +393,24 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if md5Sum != "" {
 		w.Header().Set("ETag", "\""+md5Sum+"\"")
 	}
-	encodedSuccessResponse := encodeResponse(PostResponse{
-		Location: getObjectLocation(bucket, object), // TODO Full URL is preferred
-		Bucket:   bucket,
-		Key:      object,
-		ETag:     md5Sum,
-	})
 
+	// TODO full URL is preferred.
+	w.Header().Set("Location", getObjectLocation(bucket, object))
+
+	// Set common headers.
 	setCommonHeaders(w)
 
-	writeSuccessResponse(w, encodedSuccessResponse)
+	// Write successful response.
+	writeSuccessNoContent(w)
 
-	// Fetch object info for notifications.
-	objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
-	if err != nil {
-		errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
-		return
-	}
+	if globalEventNotifier.IsBucketNotificationSet(bucket) {
+		// Fetch object info for notifications.
+		objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
+		if err != nil {
+			errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
+			return
+		}
 
-	if eventN.IsBucketNotificationSet(bucket) {
 		// Notify object created event.
 		eventNotify(eventData{
 			Type:    ObjectCreatedPost,

cmd/bucket-notification-datatypes.go

@@ -32,33 +32,30 @@ type keyFilter struct {
 	FilterRules []filterRule `xml:"FilterRule,omitempty"`
 }
 
-// Queue SQS configuration.
-type queueConfig struct {
+// Common elements of service notification.
+type serviceConfig struct {
 	Events []string `xml:"Event"`
 	Filter struct {
 		Key keyFilter `xml:"S3Key,omitempty"`
 	}
-	ID       string `xml:"Id"`
+	ID string `xml:"Id"`
+}
+
+// Queue SQS configuration.
+type queueConfig struct {
+	serviceConfig
 	QueueARN string `xml:"Queue"`
 }
 
 // Topic SNS configuration, this is a compliance field not used by minio yet.
 type topicConfig struct {
-	Events []string `xml:"Event"`
-	Filter struct {
-		Key keyFilter `xml:"S3Key"`
-	}
-	ID       string `xml:"Id"`
+	serviceConfig
 	TopicARN string `xml:"Topic"`
 }
 
 // Lambda function configuration, this is a compliance field not used by minio yet.
 type lambdaConfig struct {
-	Events []string `xml:"Event"`
-	Filter struct {
-		Key keyFilter `xml:"S3Key,omitempty"`
-	}
-	ID        string `xml:"Id"`
+	serviceConfig
 	LambdaARN string `xml:"CloudFunction"`
 }
 

cmd/bucket-notification-handlers.go

@@ -88,7 +88,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	_, err := api.ObjectAPI.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err, "Unable to bucket info.")
+		errorIf(err, "Unable to find bucket info.")
 		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 		return
 	}
@@ -141,7 +141,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	}
 
 	// Set bucket notification config.
-	eventN.SetBucketNotificationConfig(bucket, &notificationCfg)
+	globalEventNotifier.SetBucketNotificationConfig(bucket, &notificationCfg)
 
 	// Success.
 	writeSuccessResponse(w, nil)
@@ -227,7 +227,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	notificationCfg := eventN.GetBucketNotificationConfig(bucket)
+	notificationCfg := globalEventNotifier.GetBucketNotificationConfig(bucket)
 	if notificationCfg == nil {
 		writeErrorResponse(w, r, ErrARNNotification, r.URL.Path)
 		return
@@ -249,9 +249,9 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	defer close(nEventCh)
 
 	// Set sns target.
-	eventN.SetSNSTarget(topicARN, nEventCh)
+	globalEventNotifier.SetSNSTarget(topicARN, nEventCh)
 	// Remove sns listener after the writer has closed or the client disconnected.
-	defer eventN.RemoveSNSTarget(topicARN, nEventCh)
+	defer globalEventNotifier.RemoveSNSTarget(topicARN, nEventCh)
 
 	// Start sending bucket notifications.
 	sendBucketNotification(w, nEventCh)

cmd/bucket-notification-utils.go

@@ -266,17 +266,68 @@ func validateTopicConfigs(topicConfigs []topicConfig) APIErrorCode {
 	return ErrNone
 }
 
+// Check all the queue configs for any duplicates.
+func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
+	configMaps := make(map[string]int)
+
+	// Navigate through each configs and count the entries.
+	for _, config := range configs {
+		configMaps[config.QueueARN]++
+	}
+
+	// Validate if there are any duplicate counts.
+	for _, count := range configMaps {
+		if count != 1 {
+			return ErrOverlappingConfigs
+		}
+	}
+
+	// Success.
+	return ErrNone
+}
+
+// Check all the topic configs for any duplicates.
+func checkDuplicateTopicConfigs(configs []topicConfig) APIErrorCode {
+	configMaps := make(map[string]int)
+
+	// Navigate through each configs and count the entries.
+	for _, config := range configs {
+		configMaps[config.TopicARN]++
+	}
+
+	// Validate if there are any duplicate counts.
+	for _, count := range configMaps {
+		if count != 1 {
+			return ErrOverlappingConfigs
+		}
+	}
+
+	// Success.
+	return ErrNone
+}
+
 // Validates all the bucket notification configuration for their validity,
 // if one of the config is malformed or has invalid data it is rejected.
 // Configuration is never applied partially.
 func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
+	// Validate all queue configs.
 	if s3Error := validateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
 		return s3Error
 	}
+	// Validate all topic configs.
 	if s3Error := validateTopicConfigs(nConfig.TopicConfigs); s3Error != ErrNone {
 		return s3Error
 	}
 
+	// Check for duplicate queue configs.
+	if s3Error := checkDuplicateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
+		return s3Error
+	}
+	// Check for duplicate topic configs.
+	if s3Error := checkDuplicateTopicConfigs(nConfig.TopicConfigs); s3Error != ErrNone {
+		return s3Error
+	}
+
 	// Add validation for other configurations.
 	return ErrNone
 }

cmd/bucket-policy-handlers.go

@@ -70,12 +70,12 @@ func bucketPolicyActionMatch(action string, statement policyStatement) bool {
 
 // Match function matches wild cards in 'pattern' for resource.
 func resourceMatch(pattern, resource string) bool {
-	return wildcard.MatchExtended(pattern, resource)
+	return wildcard.Match(pattern, resource)
 }
 
 // Match function matches wild cards in 'pattern' for action.
 func actionMatch(pattern, action string) bool {
-	return wildcard.Match(pattern, action)
+	return wildcard.MatchSimple(pattern, action)
 }
 
 // Verify if given resource matches with policy statement.

cmd/bucket-policy.go

@@ -116,9 +116,9 @@ func getOldBucketsConfigPath() (string, error) {
 	return path.Join(configPath, "buckets"), nil
 }
 
-// readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
-// if bucket policy is not found. This function also parses the bucket policy into an object.
-func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
+// readBucketPolicyJSON - reads bucket policy for an input bucket, returns BucketPolicyNotFound
+// if bucket policy is not found.
+func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader io.Reader, err error) {
 	// Verify bucket is valid.
 	if !IsValidBucketName(bucket) {
 		return nil, BucketNameInvalid{Bucket: bucket}
@@ -139,9 +139,22 @@ func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error)
 		}
 		return nil, err
 	}
+
+	return &buffer, nil
+}
+
+// readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
+// if bucket policy is not found. This function also parses the bucket policy into an object.
+func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
+	// Read bucket policy JSON.
+	bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
+	if err != nil {
+		return nil, err
+	}
+
 	// Parse the saved policy.
 	var policy = &bucketPolicy{}
-	err = parseBucketPolicy(&buffer, policy)
+	err = parseBucketPolicy(bucketPolicyReader, policy)
 	if err != nil {
 		return nil, err
 

cmd/checkport.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"fmt"
 	"net"
+	"os"
+	"syscall"
 )
 
 // Make sure that none of the other processes are listening on the
@@ -35,7 +37,13 @@ func checkPortAvailability(port int) error {
 	for _, n := range network {
 		l, err := net.Listen(n, fmt.Sprintf(":%d", port))
 		if err != nil {
-			return err
+			if isAddrInUse(err) {
+				// Return error if another process is listening on the
+				// same port.
+				return err
+			}
+			// Ignore any other error (ex. EAFNOSUPPORT)
+			continue
 		}
 
 		// look for error so we don't have dangling connection
@@ -46,3 +54,18 @@ func checkPortAvailability(port int) error {
 
 	return nil
 }
+
+// Return true if err is "address already in use" error.
+// syscall.EADDRINUSE is available on all OSes.
+func isAddrInUse(err error) bool {
+	if opErr, ok := err.(*net.OpError); ok {
+		if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
+			if errno, ok := sysErr.Err.(syscall.Errno); ok {
+				if errno == syscall.EADDRINUSE {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

cmd/config-migrate.go

@@ -36,6 +36,8 @@ func migrateConfig() {
 	migrateV4ToV5()
 	// Migrate version '5' to '6.
 	migrateV5ToV6()
+	// Migrate version '6' to '7'.
+	migrateV6ToV7()
 }
 
 // Version '1' is not supported anymore and deprecated, safe to delete.
@@ -151,6 +153,46 @@ func migrateV3ToV4() {
 	console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
 }
 
+// Version '4' to '5' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV4ToV5() {
+	cv4, err := loadConfigV4()
+	if err != nil && os.IsNotExist(err) {
+		return
+	}
+	fatalIf(err, "Unable to load config version ‘4’.")
+	if cv4.Version != "4" {
+		return
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &configV5{}
+	srvConfig.Version = "5"
+	srvConfig.Credential = cv4.Credential
+	srvConfig.Region = cv4.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = "us-east-1"
+	}
+	srvConfig.Logger.Console = cv4.Logger.Console
+	srvConfig.Logger.File = cv4.Logger.File
+	srvConfig.Logger.Syslog = cv4.Logger.Syslog
+	srvConfig.Logger.AMQP.Enable = false
+	srvConfig.Logger.ElasticSearch.Enable = false
+	srvConfig.Logger.Redis.Enable = false
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err, "Unable to initialize the quick config.")
+	configFile, err := getConfigFile()
+	fatalIf(err, "Unable to get config file.")
+
+	err = qc.Save(configFile)
+	fatalIf(err, "Failed to migrate config from ‘"+cv4.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+
+	console.Println("Migration from version ‘" + cv4.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}
+
 // Version '5' to '6' migrates config, removes previous fields related
 // to backend types and server address. This change further simplifies
 // the config for future additions.
@@ -165,8 +207,8 @@ func migrateV5ToV6() {
 	}
 
 	// Save only the new fields, ignore the rest.
-	srvConfig := &serverConfigV6{}
-	srvConfig.Version = globalMinioConfigVersion
+	srvConfig := &configV6{}
+	srvConfig.Version = "6"
 	srvConfig.Credential = cv5.Credential
 	srvConfig.Region = cv5.Region
 	if srvConfig.Region == "" {
@@ -176,6 +218,7 @@ func migrateV5ToV6() {
 	srvConfig.Logger.Console = cv5.Logger.Console
 	srvConfig.Logger.File = cv5.Logger.File
 	srvConfig.Logger.Syslog = cv5.Logger.Syslog
+
 	srvConfig.Notify.AMQP = map[string]amqpNotify{
 		"1": {
 			Enable:      cv5.Logger.AMQP.Enable,
@@ -217,34 +260,49 @@ func migrateV5ToV6() {
 	console.Println("Migration from version ‘" + cv5.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
 }
 
-// Version '4' to '5' migrates config, removes previous fields related
+// Version '6' to '7' migrates config, removes previous fields related
 // to backend types and server address. This change further simplifies
 // the config for future additions.
-func migrateV4ToV5() {
-	cv4, err := loadConfigV4()
+func migrateV6ToV7() {
+	cv6, err := loadConfigV6()
 	if err != nil && os.IsNotExist(err) {
 		return
 	}
-	fatalIf(err, "Unable to load config version ‘4’.")
-	if cv4.Version != "4" {
+	fatalIf(err, "Unable to load config version ‘6’.")
+	if cv6.Version != "6" {
 		return
 	}
 
 	// Save only the new fields, ignore the rest.
-	srvConfig := &configV5{}
+	srvConfig := &serverConfigV7{}
 	srvConfig.Version = globalMinioConfigVersion
-	srvConfig.Credential = cv4.Credential
-	srvConfig.Region = cv4.Region
+	srvConfig.Credential = cv6.Credential
+	srvConfig.Region = cv6.Region
 	if srvConfig.Region == "" {
 		// Region needs to be set for AWS Signature Version 4.
 		srvConfig.Region = "us-east-1"
 	}
-	srvConfig.Logger.Console = cv4.Logger.Console
-	srvConfig.Logger.File = cv4.Logger.File
-	srvConfig.Logger.Syslog = cv4.Logger.Syslog
-	srvConfig.Logger.AMQP.Enable = false
-	srvConfig.Logger.ElasticSearch.Enable = false
-	srvConfig.Logger.Redis.Enable = false
+	srvConfig.Logger.Console = cv6.Logger.Console
+	srvConfig.Logger.File = cv6.Logger.File
+	srvConfig.Logger.Syslog = cv6.Logger.Syslog
+	srvConfig.Notify.AMQP = make(map[string]amqpNotify)
+	srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+	srvConfig.Notify.Redis = make(map[string]redisNotify)
+	if len(cv6.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP["1"] = amqpNotify{}
+	} else {
+		srvConfig.Notify.AMQP = cv6.Notify.AMQP
+	}
+	if len(cv6.Notify.ElasticSearch) == 0 {
+		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{}
+	} else {
+		srvConfig.Notify.ElasticSearch = cv6.Notify.ElasticSearch
+	}
+	if len(cv6.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis["1"] = redisNotify{}
+	} else {
+		srvConfig.Notify.Redis = cv6.Notify.Redis
+	}
 
 	qc, err := quick.New(srvConfig)
 	fatalIf(err, "Unable to initialize the quick config.")
@@ -252,7 +310,7 @@ func migrateV4ToV5() {
 	fatalIf(err, "Unable to get config file.")
 
 	err = qc.Save(configFile)
-	fatalIf(err, "Failed to migrate config from ‘"+cv4.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
+	fatalIf(err, "Failed to migrate config from ‘"+cv6.Version+"’ to ‘"+srvConfig.Version+"’ failed.")
 
-	console.Println("Migration from version ‘" + cv4.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+	console.Println("Migration from version ‘" + cv6.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
 }

cmd/config-old.go

@@ -274,3 +274,39 @@ func loadConfigV5() (*configV5, error) {
 	}
 	return c, nil
 }
+
+// configV6 server configuration version '6'.
+type configV6 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger logger `json:"logger"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// loadConfigV6 load config version '6'.
+func loadConfigV6() (*configV6, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	if _, err = os.Stat(configFile); err != nil {
+		return nil, err
+	}
+	c := &configV6{}
+	c.Version = "6"
+	qc, err := quick.New(c)
+	if err != nil {
+		return nil, err
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

cmd/config-v7.go

@@ -23,8 +23,8 @@ import (
 	"github.com/minio/minio/pkg/quick"
 )
 
-// serverConfigV6 server configuration version '5'.
-type serverConfigV6 struct {
+// serverConfigV7 server configuration version '7'.
+type serverConfigV7 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
@@ -45,7 +45,7 @@ type serverConfigV6 struct {
 func initConfig() error {
 	if !isConfigFileExists() {
 		// Initialize server config.
-		srvCfg := &serverConfigV6{}
+		srvCfg := &serverConfigV7{}
 		srvCfg.Version = globalMinioConfigVersion
 		srvCfg.Region = "us-east-1"
 		srvCfg.Credential = mustGenAccessKeys()
@@ -55,6 +55,14 @@ func initConfig() error {
 			Enable: true,
 			Level:  "fatal",
 		}
+
+		// Make sure to initialize notification configs.
+		srvCfg.Notify.AMQP = make(map[string]amqpNotify)
+		srvCfg.Notify.AMQP["1"] = amqpNotify{}
+		srvCfg.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
+		srvCfg.Notify.ElasticSearch["1"] = elasticSearchNotify{}
+		srvCfg.Notify.Redis = make(map[string]redisNotify)
+		srvCfg.Notify.Redis["1"] = redisNotify{}
 		srvCfg.rwMutex = &sync.RWMutex{}
 
 		// Create config path.
@@ -76,7 +84,7 @@ func initConfig() error {
 	if _, err = os.Stat(configFile); err != nil {
 		return err
 	}
-	srvCfg := &serverConfigV6{}
+	srvCfg := &serverConfigV7{}
 	srvCfg.Version = globalMinioConfigVersion
 	srvCfg.rwMutex = &sync.RWMutex{}
 	qc, err := quick.New(srvCfg)
@@ -95,10 +103,10 @@ func initConfig() error {
 }
 
 // serverConfig server config.
-var serverConfig *serverConfigV6
+var serverConfig *serverConfigV7
 
 // GetVersion get current config version.
-func (s serverConfigV6) GetVersion() string {
+func (s serverConfigV7) GetVersion() string {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Version
@@ -106,135 +114,135 @@ func (s serverConfigV6) GetVersion() string {
 
 /// Logger related.
 
-func (s *serverConfigV6) SetAMQPNotifyByID(accountID string, amqpn amqpNotify) {
+func (s *serverConfigV7) SetAMQPNotifyByID(accountID string, amqpn amqpNotify) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Notify.AMQP[accountID] = amqpn
 }
 
-func (s serverConfigV6) GetAMQP() map[string]amqpNotify {
+func (s serverConfigV7) GetAMQP() map[string]amqpNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.AMQP
 }
 
 // GetAMQPNotify get current AMQP logger.
-func (s serverConfigV6) GetAMQPNotifyByID(accountID string) amqpNotify {
+func (s serverConfigV7) GetAMQPNotifyByID(accountID string) amqpNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.AMQP[accountID]
 }
 
-func (s *serverConfigV6) SetElasticSearchNotifyByID(accountID string, esNotify elasticSearchNotify) {
+func (s *serverConfigV7) SetElasticSearchNotifyByID(accountID string, esNotify elasticSearchNotify) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Notify.ElasticSearch[accountID] = esNotify
 }
 
-func (s serverConfigV6) GetElasticSearch() map[string]elasticSearchNotify {
+func (s serverConfigV7) GetElasticSearch() map[string]elasticSearchNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.ElasticSearch
 }
 
 // GetElasticSearchNotify get current ElasicSearch logger.
-func (s serverConfigV6) GetElasticSearchNotifyByID(accountID string) elasticSearchNotify {
+func (s serverConfigV7) GetElasticSearchNotifyByID(accountID string) elasticSearchNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.ElasticSearch[accountID]
 }
 
-func (s *serverConfigV6) SetRedisNotifyByID(accountID string, rNotify redisNotify) {
+func (s *serverConfigV7) SetRedisNotifyByID(accountID string, rNotify redisNotify) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Notify.Redis[accountID] = rNotify
 }
 
-func (s serverConfigV6) GetRedis() map[string]redisNotify {
+func (s serverConfigV7) GetRedis() map[string]redisNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.Redis
 }
 
 // GetRedisNotify get current Redis logger.
-func (s serverConfigV6) GetRedisNotifyByID(accountID string) redisNotify {
+func (s serverConfigV7) GetRedisNotifyByID(accountID string) redisNotify {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Notify.Redis[accountID]
 }
 
 // SetFileLogger set new file logger.
-func (s *serverConfigV6) SetFileLogger(flogger fileLogger) {
+func (s *serverConfigV7) SetFileLogger(flogger fileLogger) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Logger.File = flogger
 }
 
 // GetFileLogger get current file logger.
-func (s serverConfigV6) GetFileLogger() fileLogger {
+func (s serverConfigV7) GetFileLogger() fileLogger {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Logger.File
 }
 
 // SetConsoleLogger set new console logger.
-func (s *serverConfigV6) SetConsoleLogger(clogger consoleLogger) {
+func (s *serverConfigV7) SetConsoleLogger(clogger consoleLogger) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Logger.Console = clogger
 }
 
 // GetConsoleLogger get current console logger.
-func (s serverConfigV6) GetConsoleLogger() consoleLogger {
+func (s serverConfigV7) GetConsoleLogger() consoleLogger {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Logger.Console
 }
 
 // SetSyslogLogger set new syslog logger.
-func (s *serverConfigV6) SetSyslogLogger(slogger syslogLogger) {
+func (s *serverConfigV7) SetSyslogLogger(slogger syslogLogger) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Logger.Syslog = slogger
 }
 
 // GetSyslogLogger get current syslog logger.
-func (s *serverConfigV6) GetSyslogLogger() syslogLogger {
+func (s *serverConfigV7) GetSyslogLogger() syslogLogger {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Logger.Syslog
 }
 
 // SetRegion set new region.
-func (s *serverConfigV6) SetRegion(region string) {
+func (s *serverConfigV7) SetRegion(region string) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Region = region
 }
 
 // GetRegion get current region.
-func (s serverConfigV6) GetRegion() string {
+func (s serverConfigV7) GetRegion() string {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Region
 }
 
 // SetCredentials set new credentials.
-func (s *serverConfigV6) SetCredential(creds credential) {
+func (s *serverConfigV7) SetCredential(creds credential) {
 	s.rwMutex.Lock()
 	defer s.rwMutex.Unlock()
 	s.Credential = creds
 }
 
 // GetCredentials get current credentials.
-func (s serverConfigV6) GetCredential() credential {
+func (s serverConfigV7) GetCredential() credential {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 	return s.Credential
 }
 
 // Save config.
-func (s serverConfigV6) Save() error {
+func (s serverConfigV7) Save() error {
 	s.rwMutex.RLock()
 	defer s.rwMutex.RUnlock()
 

cmd/config-v7_test.go

@@ -0,0 +1,107 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestServerConfig(t *testing.T) {
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	if serverConfig.GetRegion() != "us-east-1" {
+		t.Errorf("Expecting region `us-east-1` found %s", serverConfig.GetRegion())
+	}
+
+	// Set new region and verify.
+	serverConfig.SetRegion("us-west-1")
+	if serverConfig.GetRegion() != "us-west-1" {
+		t.Errorf("Expecting region `us-west-1` found %s", serverConfig.GetRegion())
+	}
+
+	// Set new amqp notification id.
+	serverConfig.SetAMQPNotifyByID("2", amqpNotify{})
+	savedNotifyCfg1 := serverConfig.GetAMQPNotifyByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg1, amqpNotify{}) {
+		t.Errorf("Expecting AMQP config %#v found %#v", amqpNotify{}, savedNotifyCfg1)
+	}
+
+	// Set new elastic search notification id.
+	serverConfig.SetElasticSearchNotifyByID("2", elasticSearchNotify{})
+	savedNotifyCfg2 := serverConfig.GetElasticSearchNotifyByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg2, elasticSearchNotify{}) {
+		t.Errorf("Expecting Elasticsearch config %#v found %#v", elasticSearchNotify{}, savedNotifyCfg2)
+	}
+
+	// Set new redis notification id.
+	serverConfig.SetRedisNotifyByID("2", redisNotify{})
+	savedNotifyCfg3 := serverConfig.GetRedisNotifyByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg3, redisNotify{}) {
+		t.Errorf("Expecting Redis config %#v found %#v", redisNotify{}, savedNotifyCfg3)
+	}
+
+	// Set new console logger.
+	serverConfig.SetConsoleLogger(consoleLogger{
+		Enable: true,
+	})
+	consoleCfg := serverConfig.GetConsoleLogger()
+	if !reflect.DeepEqual(consoleCfg, consoleLogger{Enable: true}) {
+		t.Errorf("Expecting console logger config %#v found %#v", consoleLogger{Enable: true}, consoleCfg)
+	}
+
+	// Set new file logger.
+	serverConfig.SetFileLogger(fileLogger{
+		Enable: true,
+	})
+	fileCfg := serverConfig.GetFileLogger()
+	if !reflect.DeepEqual(fileCfg, fileLogger{Enable: true}) {
+		t.Errorf("Expecting file logger config %#v found %#v", fileLogger{Enable: true}, consoleCfg)
+	}
+
+	// Set new syslog logger.
+	serverConfig.SetSyslogLogger(syslogLogger{
+		Enable: true,
+	})
+	sysLogCfg := serverConfig.GetSyslogLogger()
+	if !reflect.DeepEqual(sysLogCfg, syslogLogger{Enable: true}) {
+		t.Errorf("Expecting syslog logger config %#v found %#v", syslogLogger{Enable: true}, sysLogCfg)
+	}
+
+	// Match version.
+	if serverConfig.GetVersion() != globalMinioConfigVersion {
+		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), globalMinioConfigVersion)
+	}
+
+	// Attempt to save.
+	if err := serverConfig.Save(); err != nil {
+		t.Fatalf("Unable to save updated config file %s", err)
+	}
+
+	// Do this only once here.
+	setGlobalConfigPath(rootPath)
+
+	// Initialize server config.
+	if err := initConfig(); err != nil {
+		t.Fatalf("Unable to initialize from updated config file %s", err)
+	}
+}

cmd/control-heal-main.go

@@ -0,0 +1,120 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net/rpc"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/minio/cli"
+)
+
+var healCmd = cli.Command{
+	Name:   "heal",
+	Usage:  "To heal objects.",
+	Action: healControl,
+	CustomHelpTemplate: `NAME:
+  minio control {{.Name}} - {{.Usage}}
+
+USAGE:
+  minio control {{.Name}}
+
+EAMPLES:
+  1. Heal an object.
+     $ minio control {{.Name}} http://localhost:9000/songs/classical/western/piano.mp3
+
+  2. Heal all objects in a bucket recursively.
+     $ minio control {{.Name}}  http://localhost:9000/songs
+
+  3. Heall all objects with a given prefix recursively.
+     $ minio control {{.Name}} http://localhost:9000/songs/classical/
+`,
+}
+
+// "minio control heal" entry point.
+func healControl(ctx *cli.Context) {
+	// Parse bucket and object from url.URL.Path
+	parseBucketObject := func(path string) (bucketName string, objectName string) {
+		splits := strings.SplitN(path, string(slashSeparator), 3)
+		switch len(splits) {
+		case 0, 1:
+			bucketName = ""
+			objectName = ""
+		case 2:
+			bucketName = splits[1]
+			objectName = ""
+		case 3:
+			bucketName = splits[1]
+			objectName = splits[2]
+
+		}
+		return bucketName, objectName
+	}
+
+	if len(ctx.Args()) != 1 {
+		cli.ShowCommandHelpAndExit(ctx, "heal", 1)
+	}
+
+	parsedURL, err := url.Parse(ctx.Args()[0])
+	fatalIf(err, "Unable to parse URL")
+
+	bucketName, objectName := parseBucketObject(parsedURL.Path)
+	if bucketName == "" {
+		cli.ShowCommandHelpAndExit(ctx, "heal", 1)
+	}
+
+	client, err := rpc.DialHTTPPath("tcp", parsedURL.Host, path.Join(reservedBucket, controlPath))
+	fatalIf(err, "Unable to connect to %s", parsedURL.Host)
+
+	// If object does not have trailing "/" then it's an object, hence heal it.
+	if objectName != "" && !strings.HasSuffix(objectName, slashSeparator) {
+		fmt.Printf("Healing : /%s/%s", bucketName, objectName)
+		args := &HealObjectArgs{bucketName, objectName}
+		reply := &HealObjectReply{}
+		err = client.Call("Control.HealObject", args, reply)
+		fatalIf(err, "RPC Control.HealObject call failed")
+		fmt.Println()
+		return
+	}
+
+	// Recursively list and heal the objects.
+	prefix := objectName
+	marker := ""
+	for {
+		args := HealListArgs{bucketName, prefix, marker, "", 1000}
+		reply := &HealListReply{}
+		err = client.Call("Control.ListObjectsHeal", args, reply)
+		fatalIf(err, "RPC Heal.ListObjects call failed")
+
+		// Heal the objects returned in the ListObjects reply.
+		for _, obj := range reply.Objects {
+			fmt.Printf("Healing : /%s/%s", bucketName, obj)
+			reply := &HealObjectReply{}
+			err = client.Call("Control.HealObject", HealObjectArgs{bucketName, obj}, reply)
+			fatalIf(err, "RPC Heal.HealObject call failed")
+			fmt.Println()
+		}
+		if !reply.IsTruncated {
+			// End of listing.
+			break
+		}
+		marker = reply.NextMarker
+	}
+}

cmd/control-main.go

@@ -16,15 +16,7 @@
 
 package cmd
 
-import (
-	"fmt"
-	"strings"
-
-	"net/rpc"
-	"net/url"
-
-	"github.com/minio/cli"
-)
+import "github.com/minio/cli"
 
 // "minio control" command.
 var controlCmd = cli.Command{
@@ -33,102 +25,28 @@ var controlCmd = cli.Command{
 	Action: mainControl,
 	Subcommands: []cli.Command{
 		healCmd,
+		shutdownCmd,
 	},
-}
-
-func mainControl(c *cli.Context) {
-	cli.ShowCommandHelp(c, "")
-}
-
-var healCmd = cli.Command{
-	Name:   "heal",
-	Usage:  "To heal objects.",
-	Action: healControl,
 	CustomHelpTemplate: `NAME:
-  minio control {{.Name}} - {{.Usage}}
+   {{.Name}} - {{.Usage}}
 
 USAGE:
-  minio control {{.Name}}
+   {{.Name}} [FLAGS] COMMAND
 
-EAMPLES:
-  1. Heal an object.
-     $ minio control {{.Name}} http://localhost:9000/songs/classical/western/piano.mp3
-
-  2. Heal all objects in a bucket recursively.
-     $ minio control {{.Name}}  http://localhost:9000/songs
-
-  3. Heall all objects with a given prefix recursively.
-     $ minio control {{.Name}} http://localhost:9000/songs/classical/
+FLAGS:
+  {{range .Flags}}{{.}}
+  {{end}}
+COMMANDS:
+   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
+   {{end}}
 `,
 }
 
-// "minio control heal" entry point.
-func healControl(c *cli.Context) {
-	// Parse bucket and object from url.URL.Path
-	parseBucketObject := func(path string) (bucketName string, objectName string) {
-		splits := strings.SplitN(path, string(slashSeparator), 3)
-		switch len(splits) {
-		case 0, 1:
-			bucketName = ""
-			objectName = ""
-		case 2:
-			bucketName = splits[1]
-			objectName = ""
-		case 3:
-			bucketName = splits[1]
-			objectName = splits[2]
-
-		}
-		return bucketName, objectName
-	}
-
-	if len(c.Args()) != 1 {
-		cli.ShowCommandHelpAndExit(c, "heal", 1)
-	}
-
-	parsedURL, err := url.ParseRequestURI(c.Args()[0])
-	fatalIf(err, "Unable to parse URL")
-
-	bucketName, objectName := parseBucketObject(parsedURL.Path)
-	if bucketName == "" {
-		cli.ShowCommandHelpAndExit(c, "heal", 1)
-	}
-
-	client, err := rpc.DialHTTPPath("tcp", parsedURL.Host, healPath)
-	fatalIf(err, "Unable to connect to %s", parsedURL.Host)
-
-	// If object does not have trailing "/" then it's an object, hence heal it.
-	if objectName != "" && !strings.HasSuffix(objectName, slashSeparator) {
-		fmt.Printf("Healing : /%s/%s", bucketName, objectName)
-		args := &HealObjectArgs{bucketName, objectName}
-		reply := &HealObjectReply{}
-		err = client.Call("Heal.HealObject", args, reply)
-		fatalIf(err, "RPC Heal.HealObject call failed")
-		fmt.Println()
-		return
-	}
-
-	// Recursively list and heal the objects.
-	prefix := objectName
-	marker := ""
-	for {
-		args := HealListArgs{bucketName, prefix, marker, "", 1000}
-		reply := &HealListReply{}
-		err = client.Call("Heal.ListObjects", args, reply)
-		fatalIf(err, "RPC Heal.ListObjects call failed")
-
-		// Heal the objects returned in the ListObjects reply.
-		for _, obj := range reply.Objects {
-			fmt.Printf("Healing : /%s/%s", bucketName, obj)
-			reply := &HealObjectReply{}
-			err = client.Call("Heal.HealObject", HealObjectArgs{bucketName, obj}, reply)
-			fatalIf(err, "RPC Heal.HealObject call failed")
-			fmt.Println()
-		}
-		if !reply.IsTruncated {
-			// End of listing.
-			break
-		}
-		marker = reply.NextMarker
+func mainControl(ctx *cli.Context) {
+	if ctx.Args().First() != "" { // command help.
+		cli.ShowCommandHelp(ctx, ctx.Args().First())
+	} else {
+		// command with Subcommands is an App.
+		cli.ShowAppHelp(ctx)
 	}
 }

cmd/control-shutdown-main.go

@@ -0,0 +1,68 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/rpc"
+	"net/url"
+	"path"
+
+	"github.com/minio/cli"
+)
+
+var shutdownCmd = cli.Command{
+	Name:   "shutdown",
+	Usage:  "Shutdown or restart the server.",
+	Action: shutdownControl,
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "restart",
+			Usage: "Restart the server.",
+		},
+	},
+	CustomHelpTemplate: `NAME:
+  minio control {{.Name}} - {{.Usage}}
+
+USAGE:
+  minio control {{.Name}} http://localhost:9000/
+
+EAMPLES:
+  1. Shutdown the server:
+    $ minio control shutdown http://localhost:9000/
+
+  2. Reboot the server:
+    $ minio control shutdown --restart http://localhost:9000/
+`,
+}
+
+// "minio control shutdown" entry point.
+func shutdownControl(c *cli.Context) {
+	if len(c.Args()) != 1 {
+		cli.ShowCommandHelpAndExit(c, "shutdown", 1)
+	}
+
+	parsedURL, err := url.ParseRequestURI(c.Args()[0])
+	fatalIf(err, "Unable to parse URL")
+
+	client, err := rpc.DialHTTPPath("tcp", parsedURL.Host, path.Join(reservedBucket, controlPath))
+	fatalIf(err, "Unable to connect to %s", parsedURL.Host)
+
+	args := &ShutdownArgs{Reboot: c.Bool("restart")}
+	reply := &ShutdownReply{}
+	err = client.Call("Control.Shutdown", args, reply)
+	fatalIf(err, "RPC Control.Shutdown call failed")
+}

cmd/controller-handlers.go

@@ -16,30 +16,6 @@
 
 package cmd
 
-import (
-	"net/rpc"
-
-	router "github.com/gorilla/mux"
-)
-
-// Routes paths for "minio control" commands.
-const (
-	controlRPCPath = reservedBucket + "/control"
-	healPath       = controlRPCPath + "/heal"
-)
-
-// Register control RPC handlers.
-func registerControlRPCRouter(mux *router.Router, objAPI ObjectLayer) {
-	healRPCServer := rpc.NewServer()
-	healRPCServer.RegisterName("Heal", &healHandler{objAPI})
-	mux.Path(healPath).Handler(healRPCServer)
-}
-
-// Handler for object healing.
-type healHandler struct {
-	ObjectAPI ObjectLayer
-}
-
 // HealListArgs - argument for ListObjects RPC.
 type HealListArgs struct {
 	Bucket    string
@@ -56,9 +32,13 @@ type HealListReply struct {
 	Objects     []string
 }
 
-// ListObjects - list objects.
-func (h healHandler) ListObjects(arg *HealListArgs, reply *HealListReply) error {
-	info, err := h.ObjectAPI.ListObjectsHeal(arg.Bucket, arg.Prefix, arg.Marker, arg.Delimiter, arg.MaxKeys)
+// ListObjects - list all objects that needs healing.
+func (c *controllerAPIHandlers) ListObjectsHeal(arg *HealListArgs, reply *HealListReply) error {
+	objAPI := c.ObjectAPI
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+	info, err := objAPI.ListObjectsHeal(arg.Bucket, arg.Prefix, arg.Marker, arg.Delimiter, arg.MaxKeys)
 	if err != nil {
 		return err
 	}
@@ -80,6 +60,29 @@ type HealObjectArgs struct {
 type HealObjectReply struct{}
 
 // HealObject - heal the object.
-func (h healHandler) HealObject(arg *HealObjectArgs, reply *HealObjectReply) error {
-	return h.ObjectAPI.HealObject(arg.Bucket, arg.Object)
+func (c *controllerAPIHandlers) HealObject(arg *HealObjectArgs, reply *HealObjectReply) error {
+	objAPI := c.ObjectAPI
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+	return objAPI.HealObject(arg.Bucket, arg.Object)
+}
+
+// ShutdownArgs - argument for Shutdown RPC.
+type ShutdownArgs struct {
+	Reboot bool
+}
+
+// ShutdownReply - reply by Shutdown RPC.
+type ShutdownReply struct{}
+
+// Shutdown - Shutdown the server.
+
+func (c *controllerAPIHandlers) Shutdown(arg *ShutdownArgs, reply *ShutdownReply) error {
+	if arg.Reboot {
+		globalShutdownSignalCh <- shutdownRestart
+	} else {
+		globalShutdownSignalCh <- shutdownHalt
+	}
+	return nil
 }

cmd/controller-router.go

@@ -0,0 +1,42 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/rpc"
+
+	router "github.com/gorilla/mux"
+)
+
+// Routes paths for "minio control" commands.
+const (
+	controlPath = "/controller"
+)
+
+// Register control RPC handlers.
+func registerControlRPCRouter(mux *router.Router, ctrlHandlers *controllerAPIHandlers) {
+	ctrlRPCServer := rpc.NewServer()
+	ctrlRPCServer.RegisterName("Control", ctrlHandlers)
+
+	ctrlRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter()
+	ctrlRouter.Path(controlPath).Handler(ctrlRPCServer)
+}
+
+// Handler for object healing.
+type controllerAPIHandlers struct {
+	ObjectAPI ObjectLayer
+}

cmd/erasure-readfile_test.go

@@ -112,7 +112,7 @@ func testGetReadDisks(t *testing.T, xl xlObjects) {
 			t.Errorf("test-case %d - expected nextIndex: %d, got : %d", i+1, test.nextIndex, nextIndex)
 			continue
 		}
-		if reflect.DeepEqual(test.retDisks, disks) == false {
+		if !reflect.DeepEqual(test.retDisks, disks) {
 			t.Errorf("test-case %d : incorrect disks returned. expected %+v, got %+v", i+1, test.retDisks, disks)
 			continue
 		}

cmd/event-notifier.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"encoding/xml"
-	"errors"
 	"fmt"
 	"net/url"
 	"path"
@@ -112,7 +111,7 @@ func (en *eventNotifier) SetSNSTarget(snsARN string, listenerCh chan []Notificat
 	en.rwMutex.Lock()
 	defer en.rwMutex.Unlock()
 	if listenerCh == nil {
-		return errors.New("invalid argument")
+		return errInvalidArgument
 	}
 	en.snsTargets[snsARN] = append(en.snsTargets[snsARN], listenerCh)
 	return nil
@@ -161,7 +160,7 @@ func (en *eventNotifier) SetBucketNotificationConfig(bucket string, notification
 	en.rwMutex.Lock()
 	defer en.rwMutex.Unlock()
 	if notificationCfg == nil {
-		return errors.New("invalid argument")
+		return errInvalidArgument
 	}
 	en.notificationConfigs[bucket] = notificationCfg
 	return nil
@@ -178,8 +177,11 @@ func eventNotify(event eventData) {
 	//  - s3:ObjectCreated:CompleteMultipartUpload
 	//  - s3:ObjectRemoved:Delete
 
-	nConfig := eventN.GetBucketNotificationConfig(event.Bucket)
+	nConfig := globalEventNotifier.GetBucketNotificationConfig(event.Bucket)
 	// No bucket notifications enabled, drop the event notification.
+	if nConfig == nil {
+		return
+	}
 	if len(nConfig.QueueConfigs) == 0 && len(nConfig.TopicConfigs) == 0 && len(nConfig.LambdaConfigs) == 0 {
 		return
 	}
@@ -198,7 +200,7 @@ func eventNotify(event eventData) {
 		eventMatch := eventMatch(eventType, qConfig.Events)
 		ruleMatch := filterRuleMatch(objectName, qConfig.Filter.Key.FilterRules)
 		if eventMatch && ruleMatch {
-			targetLog := eventN.GetQueueTarget(qConfig.QueueARN)
+			targetLog := globalEventNotifier.GetQueueTarget(qConfig.QueueARN)
 			if targetLog != nil {
 				targetLog.WithFields(logrus.Fields{
 					"Records": notificationEvent,
@@ -211,7 +213,7 @@ func eventNotify(event eventData) {
 		ruleMatch := filterRuleMatch(objectName, topicConfig.Filter.Key.FilterRules)
 		eventMatch := eventMatch(eventType, topicConfig.Events)
 		if eventMatch && ruleMatch {
-			targetListeners := eventN.GetSNSTarget(topicConfig.TopicARN)
+			targetListeners := globalEventNotifier.GetSNSTarget(topicConfig.TopicARN)
 			for _, listener := range targetListeners {
 				listener <- notificationEvent
 			}
@@ -352,7 +354,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 }
 
 // Global instance of event notification queue.
-var eventN *eventNotifier
+var globalEventNotifier *eventNotifier
 
 // Initialize event notifier.
 func initEventNotifier(objAPI ObjectLayer) error {
@@ -373,7 +375,7 @@ func initEventNotifier(objAPI ObjectLayer) error {
 	}
 
 	// Inititalize event notifier queue.
-	eventN = &eventNotifier{
+	globalEventNotifier = &eventNotifier{
 		rwMutex:             &sync.RWMutex{},
 		notificationConfigs: configs,
 		queueTargets:        queueTargets,

cmd/event-notifier_test.go

@@ -16,7 +16,73 @@
 
 package cmd
 
-import "testing"
+import (
+	"reflect"
+	"testing"
+)
+
+// Tests event notify.
+func TestEventNotify(t *testing.T) {
+	ExecObjectLayerTest(t, testEventNotify)
+}
+
+func testEventNotify(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	bucketName := getRandomBucketName()
+
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	initEventNotifier(obj)
+
+	// Notify object created event.
+	eventNotify(eventData{
+		Type:   ObjectCreatedPost,
+		Bucket: bucketName,
+		ObjInfo: ObjectInfo{
+			Bucket: bucketName,
+			Name:   "object1",
+		},
+		ReqParams: map[string]string{
+			"sourceIPAddress": "localhost:1337",
+		},
+	})
+
+	if err := globalEventNotifier.SetBucketNotificationConfig(bucketName, nil); err != errInvalidArgument {
+		t.Errorf("Expected error %s, got %s", errInvalidArgument, err)
+	}
+
+	if err := globalEventNotifier.SetBucketNotificationConfig(bucketName, &notificationConfig{}); err != nil {
+		t.Errorf("Expected error to be nil, got %s", err)
+	}
+
+	if !globalEventNotifier.IsBucketNotificationSet(bucketName) {
+		t.Errorf("Notification expected to be set, but notification not set.")
+	}
+
+	nConfig := globalEventNotifier.GetBucketNotificationConfig(bucketName)
+	if !reflect.DeepEqual(nConfig, &notificationConfig{}) {
+		t.Errorf("Mismatching notification configs.")
+	}
+
+	// Notify object created event.
+	eventNotify(eventData{
+		Type:   ObjectCreatedPost,
+		Bucket: bucketName,
+		ObjInfo: ObjectInfo{
+			Bucket: bucketName,
+			Name:   "object1",
+		},
+		ReqParams: map[string]string{
+			"sourceIPAddress": "localhost:1337",
+		},
+	})
+}
 
 // Tests various forms of inititalization of event notifier.
 func TestInitEventNotifier(t *testing.T) {

cmd/faulty-disk_test.go

@@ -1,115 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-// Simulates disk returning errFaultyDisk on all methods of StorageAPI
-// interface after successCount number of successes.
-type faultyDisk struct {
-	disk         *posix
-	successCount int
-}
-
-// instantiates a faulty
-func newFaultyDisk(disk *posix, n int) *faultyDisk {
-	return &faultyDisk{disk: disk, successCount: n}
-}
-
-func (f *faultyDisk) MakeVol(volume string) (err error) {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.MakeVol(volume)
-	}
-	return errFaultyDisk
-}
-func (f *faultyDisk) ListVols() (vols []VolInfo, err error) {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.ListVols()
-	}
-	return nil, errFaultyDisk
-}
-
-func (f *faultyDisk) StatVol(volume string) (volInfo VolInfo, err error) {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.StatVol(volume)
-	}
-	return VolInfo{}, errFaultyDisk
-}
-func (f *faultyDisk) DeleteVol(volume string) (err error) {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.DeleteVol(volume)
-	}
-	return errFaultyDisk
-}
-
-func (f *faultyDisk) ListDir(volume, path string) (entries []string, err error) {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.ListDir(volume, path)
-	}
-	return []string{}, errFaultyDisk
-}
-
-func (f *faultyDisk) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.ReadFile(volume, path, offset, buf)
-	}
-	return 0, errFaultyDisk
-}
-
-func (f *faultyDisk) AppendFile(volume, path string, buf []byte) error {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.AppendFile(volume, path, buf)
-	}
-	return errFaultyDisk
-}
-
-func (f *faultyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath)
-	}
-	return errFaultyDisk
-}
-
-func (f *faultyDisk) StatFile(volume string, path string) (file FileInfo, err error) {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.StatFile(volume, path)
-	}
-	return FileInfo{}, errFaultyDisk
-}
-
-func (f *faultyDisk) DeleteFile(volume string, path string) (err error) {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.DeleteFile(volume, path)
-	}
-	return errFaultyDisk
-}
-
-func (f *faultyDisk) ReadAll(volume string, path string) (buf []byte, err error) {
-	if f.successCount > 0 {
-		f.successCount--
-		return f.disk.ReadAll(volume, path)
-	}
-	return nil, errFaultyDisk
-}

cmd/format-config-v1_test.go

@@ -66,6 +66,75 @@ func genFormatXLInvalidVersion() []*formatConfigV1 {
 	return formatConfigs
 }
 
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidFormat() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Corrupt version numbers.
+	formatConfigs[0].Format = "lx"
+	formatConfigs[3].Format = "lx"
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidXLVersion() []*formatConfigV1 {
+	jbod := make([]string, 8)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	// Corrupt version numbers.
+	formatConfigs[0].XL.Version = "10"
+	formatConfigs[3].XL.Version = "-1"
+	return formatConfigs
+}
+
+// generates a invalid format.json version for XL backend.
+func genFormatXLInvalidJBODCount() []*formatConfigV1 {
+	jbod := make([]string, 7)
+	formatConfigs := make([]*formatConfigV1, 8)
+	for index := range jbod {
+		jbod[index] = getUUID()
+	}
+	for index := range jbod {
+		formatConfigs[index] = &formatConfigV1{
+			Version: "1",
+			Format:  "xl",
+			XL: &xlFormat{
+				Version: "1",
+				Disk:    jbod[index],
+				JBOD:    jbod,
+			},
+		}
+	}
+	return formatConfigs
+}
+
 // generates a invalid format.json JBOD for XL backend.
 func genFormatXLInvalidJBOD() []*formatConfigV1 {
 	jbod := make([]string, 8)
@@ -145,18 +214,14 @@ func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
 	return formatConfigs
 }
 
-func TestFormatXLHealFreshDisks(t *testing.T) {
-	// Create an instance of xl backend.
-	obj, fsDirs, err := getXLObjectLayer()
-	if err != nil {
-		t.Fatal(err)
-	}
+func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
 
+	var err error
 	xl := obj.(xlObjects)
 
 	err = obj.MakeBucket("bucket")
 	if err != nil {
-		t.Fatal(err)
+		return []StorageAPI{}, err
 	}
 
 	bucket := "bucket"
@@ -164,34 +229,26 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 
 	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil)
 	if err != nil {
-		t.Fatal(err)
+		return []StorageAPI{}, err
 	}
 
-	/* // Now, remove two format files.. Load them and reorder
-	if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
-		t.Fatal(err)
-	}
-	if err = xl.storageDisks[11].DeleteFile(".minio.sys", "format.json"); err != nil {
-		t.Fatal(err)
-	} */
-
 	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
 	// created when minio starts
 	for i := 3; i <= 5; i++ {
 		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "tmp"); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/xl.json"); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteFile(bucket, object+"/part.1"); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 		if err = xl.storageDisks[i].DeleteVol(bucket); err != nil {
-			t.Fatal(err)
+			return []StorageAPI{}, err
 		}
 	}
 
@@ -201,14 +258,29 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
 		xl.storageDisks[5], xl.storageDisks[11]}
 
+	return permutedStorageDisks, nil
+
+}
+
+func TestFormatXLHealFreshDisks(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Error(err)
+	}
+
+	storageDisks, err := prepareFormatXLHealFreshDisks(obj)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Start healing disks
-	err = healFormatXLFreshDisks(permutedStorageDisks)
+	err = healFormatXLFreshDisks(storageDisks)
 	if err != nil {
 		t.Fatal("healing corrupted disk failed: ", err)
 	}
 
 	// Load again XL format.json to validate it
-	_, err = loadFormatXL(permutedStorageDisks)
+	_, err = loadFormatXL(storageDisks)
 	if err != nil {
 		t.Fatal("loading healed disk failed: ", err)
 	}
@@ -217,6 +289,39 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 	removeRoots(fsDirs)
 }
 
+func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Error(err)
+	}
+
+	storageDisks, err := prepareFormatXLHealFreshDisks(obj)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 16; i++ {
+		d := storageDisks[i].(*posix)
+		storageDisks[i] = &naughtyDisk{disk: d, defaultErr: errDiskNotFound}
+	}
+
+	// Load again XL format.json to validate it
+	_, err = loadFormatXL(storageDisks)
+	if err == nil {
+		t.Fatal("loading format disk error")
+	}
+
+	storageDisks[3] = nil
+	err = healFormatXLFreshDisks(storageDisks)
+	if err != nil {
+		t.Fatal("didn't get nil when one disk is offline")
+	}
+
+	// Clean all
+	removeRoots(fsDirs)
+}
+
 // Simulate XL disks creation, delete some format.json and remove the content of
 // a given disk to test healing a corrupted disk
 func TestFormatXLHealCorruptedDisks(t *testing.T) {
@@ -355,12 +460,18 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 // Wrapper for calling FormatXL tests - currently validates
 //  - valid format
 //  - unrecognized version number
+//  - unrecognized format tag
+//  - unrecognized xl version
+//  - wrong number of JBOD entries
 //  - invalid JBOD
 //  - invalid Disk uuid
 func TestFormatXL(t *testing.T) {
 	formatInputCases := [][]*formatConfigV1{
 		genFormatXLValid(),
 		genFormatXLInvalidVersion(),
+		genFormatXLInvalidFormat(),
+		genFormatXLInvalidXLVersion(),
+		genFormatXLInvalidJBODCount(),
 		genFormatXLInvalidJBOD(),
 		genFormatXLInvalidDisks(),
 		genFormatXLInvalidDisksOrder(),
@@ -389,6 +500,18 @@ func TestFormatXL(t *testing.T) {
 			formatConfigs: formatInputCases[4],
 			shouldPass:    false,
 		},
+		{
+			formatConfigs: formatInputCases[5],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[6],
+			shouldPass:    false,
+		},
+		{
+			formatConfigs: formatInputCases[7],
+			shouldPass:    false,
+		},
 	}
 
 	for i, testCase := range testCases {
@@ -443,3 +566,378 @@ func TestSavedUUIDOrder(t *testing.T) {
 		}
 	}
 }
+
+// Test initFormatXL() when disks are expected to return errors
+func TestInitFormatXLErrors(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl := obj.(xlObjects)
+
+	testStorageDisks := make([]StorageAPI, 16)
+
+	// All disks API return disk not found
+	for i := 0; i < 16; i++ {
+		d := xl.storageDisks[i].(*posix)
+		testStorageDisks[i] = &naughtyDisk{disk: d, defaultErr: errDiskNotFound}
+	}
+	if err := initFormatXL(testStorageDisks); err != errDiskNotFound {
+		t.Fatal("Got a different error: ", err)
+	}
+
+	// All disks returns disk not found in the fourth call
+	for i := 0; i < 15; i++ {
+		d := xl.storageDisks[i].(*posix)
+		testStorageDisks[i] = &naughtyDisk{disk: d, defaultErr: errDiskNotFound, errors: map[int]error{0: nil, 1: nil, 2: nil}}
+	}
+	if err := initFormatXL(testStorageDisks); err != errDiskNotFound {
+		t.Fatal("Got a different error: ", err)
+	}
+
+	// All disks are nil (disk not found)
+	for i := 0; i < 15; i++ {
+		testStorageDisks[i] = nil
+	}
+	if err := initFormatXL(testStorageDisks); err != errDiskNotFound {
+		t.Fatal("Got a different error: ", err)
+	}
+
+	removeRoots(fsDirs)
+}
+
+// Test for reduceFormatErrs()
+func TestReduceFormatErrs(t *testing.T) {
+	// No error founds
+	if err := reduceFormatErrs([]error{nil, nil, nil, nil}, 4); err != nil {
+		t.Fatal("Err should be nil, found: ", err)
+	}
+	// One corrupted format
+	if err := reduceFormatErrs([]error{nil, nil, errCorruptedFormat, nil}, 4); err != errCorruptedFormat {
+		t.Fatal("Got a differnt error: ", err)
+	}
+	// All disks unformatted
+	if err := reduceFormatErrs([]error{errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk}, 4); err != errUnformattedDisk {
+		t.Fatal("Got a differnt error: ", err)
+	}
+	// Some disks unformatted
+	if err := reduceFormatErrs([]error{nil, nil, errUnformattedDisk, errUnformattedDisk}, 4); err != errSomeDiskUnformatted {
+		t.Fatal("Got a differnt error: ", err)
+	}
+	// Some disks offline
+	if err := reduceFormatErrs([]error{nil, nil, errDiskNotFound, errUnformattedDisk}, 4); err != errSomeDiskOffline {
+		t.Fatal("Got a differnt error: ", err)
+	}
+}
+
+// Tests for genericFormatCheck()
+func TestGenericFormatCheck(t *testing.T) {
+	var errs []error
+	formatConfigs := genFormatXLInvalidJBOD()
+
+	// Some disks has corrupted formats, one faulty disk
+	errs = []error{nil, nil, errCorruptedFormat, errCorruptedFormat, errCorruptedFormat, errCorruptedFormat,
+		errCorruptedFormat, errFaultyDisk}
+	if err := genericFormatCheck(formatConfigs, errs); err != errCorruptedFormat {
+		t.Fatal("Got unexpected err: ", err)
+	}
+
+	// Many faulty disks
+	errs = []error{nil, nil, errFaultyDisk, errFaultyDisk, errFaultyDisk, errFaultyDisk,
+		errCorruptedFormat, errFaultyDisk}
+	if err := genericFormatCheck(formatConfigs, errs); err != errXLReadQuorum {
+		t.Fatal("Got unexpected err: ", err)
+	}
+
+	// All formats successfully loaded
+	errs = []error{nil, nil, nil, nil, nil, nil, nil, nil}
+	if err := genericFormatCheck(formatConfigs, errs); err == nil {
+		t.Fatalf("Should fail here")
+	}
+
+}
+
+func TestLoadFormatXLErrs(t *testing.T) {
+	// Create an instance of xl backend.
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl := obj.(xlObjects)
+
+	xl.storageDisks[11] = nil
+
+	// disk 12 returns faulty disk
+	posixDisk, ok := xl.storageDisks[12].(*posix)
+	if !ok {
+		t.Fatal("storage disk is not *posix type")
+	}
+	xl.storageDisks[10] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
+	if _, err = loadFormatXL(xl.storageDisks); err != errFaultyDisk {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+
+	removeRoots(fsDirs)
+
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+
+	// disks 0..10 returns disk not found
+	for i := 0; i <= 10; i++ {
+		posixDisk, ok := xl.storageDisks[i].(*posix)
+		if !ok {
+			t.Fatal("storage disk is not *posix type")
+		}
+		xl.storageDisks[i] = newNaughtyDisk(posixDisk, nil, errDiskNotFound)
+	}
+	if _, err = loadFormatXL(xl.storageDisks); err != errXLReadQuorum {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+
+	removeRoots(fsDirs)
+
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+
+	// disks 0..10 returns unformatted disk
+	for i := 0; i <= 10; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err = loadFormatXL(xl.storageDisks); err != errUnformattedDisk {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+
+	removeRoots(fsDirs)
+
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+
+	// disks 0..15 returns is nil (disk not found)
+	for i := 0; i < 16; i++ {
+		xl.storageDisks[i] = nil
+	}
+	if _, err := loadFormatXL(xl.storageDisks); err != errDiskNotFound {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+
+	removeRoots(fsDirs)
+}
+
+// Tests for healFormatXLCorruptedDisks() with cases which lead to errors
+func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
+	// Everything is fine, should return nil
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl := obj.(xlObjects)
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Disks 0..15 are nil
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		xl.storageDisks[i] = nil
+	}
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// One disk returns Faulty Disk
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	posixDisk, ok := xl.storageDisks[0].(*posix)
+	if !ok {
+		t.Fatal("storage disk is not *posix type")
+	}
+	xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != errFaultyDisk {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// One disk is not found, heal corrupted disks should return nil
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	xl.storageDisks[0] = nil
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Remove format.json of all disks
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Corrupted format json in one disk
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].AppendFile(".minio.sys", "format.json", []byte("corrupted data")); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err == nil {
+		t.Fatal("Should get a json parsing error, ")
+	}
+	removeRoots(fsDirs)
+}
+
+// Tests for healFormatXLFreshDisks() with cases which lead to errors
+func TestHealFormatXLFreshDisksErrs(t *testing.T) {
+	// Everything is fine, should return nil
+	obj, fsDirs, err := getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl := obj.(xlObjects)
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Disks 0..15 are nil
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		xl.storageDisks[i] = nil
+	}
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// One disk returns Faulty Disk
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	posixDisk, ok := xl.storageDisks[0].(*posix)
+	if !ok {
+		t.Fatal("storage disk is not *posix type")
+	}
+	xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != errFaultyDisk {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// One disk is not found, heal corrupted disks should return nil
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	xl.storageDisks[0] = nil
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Remove format.json of all disks
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	// Remove format.json of all disks
+	obj, fsDirs, err = getXLObjectLayer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+}
+
+// Tests for isFormatFound()
+func TestIsFormatFound(t *testing.T) {
+	formats := genFormatXLValid()
+	if found := isFormatFound(formats); !found {
+		t.Fatal("isFormatFound() should not return false")
+	}
+	formats[0] = nil
+	if found := isFormatFound(formats); found {
+		t.Fatal("isFormatFound() should not return true")
+	}
+}
+
+// Tests for isFormatNotFound()
+func TestIsFormatNotFound(t *testing.T) {
+	formats := genFormatXLValid()
+	if found := isFormatNotFound(formats); found {
+		t.Fatal("isFormatFound() should not return true")
+	}
+	formats[0] = nil
+	if found := isFormatNotFound(formats); found {
+		t.Fatal("isFormatFound() should not return true")
+	}
+	for idx := range formats {
+		formats[idx] = nil
+	}
+	if found := isFormatNotFound(formats); !found {
+		t.Fatal("isFormatFound() should not return false")
+	}
+}

cmd/fs-v1-metadata.go

@@ -77,9 +77,9 @@ func (m *fsMetaV1) AddObjectPart(partNumber int, partName string, partETag strin
 }
 
 // readFSMetadata - returns the object metadata `fs.json` content.
-func readFSMetadata(disk StorageAPI, bucket, object string) (fsMeta fsMetaV1, err error) {
+func readFSMetadata(disk StorageAPI, bucket, filePath string) (fsMeta fsMetaV1, err error) {
 	// Read all `fs.json`.
-	buf, err := disk.ReadAll(bucket, path.Join(object, fsMetaJSONFile))
+	buf, err := disk.ReadAll(bucket, filePath)
 	if err != nil {
 		return fsMetaV1{}, err
 	}
@@ -93,6 +93,19 @@ func readFSMetadata(disk StorageAPI, bucket, object string) (fsMeta fsMetaV1, er
 	return fsMeta, nil
 }
 
+// Write fsMeta to fs.json or fs-append.json.
+func writeFSMetadata(disk StorageAPI, bucket, filePath string, fsMeta fsMetaV1) (err error) {
+	tmpPath := path.Join(tmpMetaPrefix, getUUID())
+	metadataBytes, err := json.Marshal(fsMeta)
+	if err != nil {
+		return err
+	}
+	if err = disk.AppendFile(minioMetaBucket, tmpPath, metadataBytes); err != nil {
+		return err
+	}
+	return disk.RenameFile(minioMetaBucket, tmpPath, bucket, filePath)
+}
+
 // newFSMetaV1 - initializes new fsMetaV1.
 func newFSMetaV1() (fsMeta fsMetaV1) {
 	fsMeta = fsMetaV1{}
@@ -131,17 +144,18 @@ func writeFSFormatData(storage StorageAPI, fsFormat formatConfigV1) error {
 	return nil
 }
 
-// writeFSMetadata - writes `fs.json` metadata, marshals fsMeta object into json
-// and saves it to disk.
-func writeFSMetadata(storage StorageAPI, bucket, path string, fsMeta fsMetaV1) error {
-	metadataBytes, err := json.Marshal(fsMeta)
-	if err != nil {
-		return err
+// Return if the part info in uploadedParts and completeParts are same.
+func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) bool {
+	if len(uploadedParts) != len(completeParts) {
+		return false
 	}
-	if err = storage.AppendFile(bucket, path, metadataBytes); err != nil {
-		return err
+	for i := range completeParts {
+		if uploadedParts[i].Number != completeParts[i].PartNumber ||
+			uploadedParts[i].ETag != completeParts[i].ETag {
+			return false
+		}
 	}
-	return nil
+	return true
 }
 
 var extendedHeaders = []string{

cmd/fs-v1-multipart.go

@@ -235,17 +235,9 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st
 	if err = fs.writeUploadJSON(bucket, object, uploadID, initiated); err != nil {
 		return "", err
 	}
-	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
-	tempFSMetadataPath := path.Join(tmpMetaPrefix, getUUID()+"-"+fsMetaJSONFile)
-	if err = writeFSMetadata(fs.storage, minioMetaBucket, tempFSMetadataPath, fsMeta); err != nil {
-		return "", toObjectErr(err, minioMetaBucket, tempFSMetadataPath)
-	}
-	err = fs.storage.RenameFile(minioMetaBucket, tempFSMetadataPath, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
-	if err != nil {
-		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempFSMetadataPath); dErr != nil {
-			return "", toObjectErr(dErr, minioMetaBucket, tempFSMetadataPath)
-		}
-		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+	fsMetaPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, fsMetaJSONFile)
+	if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
 	// Return success.
 	return uploadID, nil
@@ -272,6 +264,127 @@ func (fs fsObjects) NewMultipartUpload(bucket, object string, meta map[string]st
 	return fs.newMultipartUpload(bucket, object, meta)
 }
 
+// Returns if a new part can be appended to fsAppendDataFile.
+func partToAppend(fsMeta fsMetaV1, fsAppendMeta fsMetaV1) (part objectPartInfo, appendNeeded bool) {
+	if len(fsMeta.Parts) == 0 {
+		return
+	}
+	// As fsAppendMeta.Parts will be sorted len(fsAppendMeta.Parts) will naturally be the next part number
+	nextPartNum := len(fsAppendMeta.Parts) + 1
+	nextPartIndex := fsMeta.ObjectPartIndex(nextPartNum)
+	if nextPartIndex == -1 {
+		return
+	}
+	return fsMeta.Parts[nextPartIndex], true
+}
+
+// Returns metadata path for the file holding info about the parts that
+// have been appended to the "append-file"
+func getFSAppendMetaPath(uploadID string) string {
+	return path.Join(tmpMetaPrefix, uploadID+".json")
+}
+
+// Returns path for the append-file.
+func getFSAppendDataPath(uploadID string) string {
+	return path.Join(tmpMetaPrefix, uploadID+".data")
+}
+
+// Append parts to fsAppendDataFile.
+func appendParts(disk StorageAPI, bucket, object, uploadID string) {
+	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
+	// fs-append.json path
+	fsAppendMetaPath := getFSAppendMetaPath(uploadID)
+	// fs.json path
+	fsMetaPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, fsMetaJSONFile)
+
+	// Lock the uploadID so that no one modifies fs.json
+	nsMutex.RLock(minioMetaBucket, uploadIDPath)
+	fsMeta, err := readFSMetadata(disk, minioMetaBucket, fsMetaPath)
+	nsMutex.RUnlock(minioMetaBucket, uploadIDPath)
+	if err != nil {
+		return
+	}
+
+	// Lock fs-append.json so that there is no parallel append to the file.
+	nsMutex.Lock(minioMetaBucket, fsAppendMetaPath)
+	defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath)
+
+	fsAppendMeta, err := readFSMetadata(disk, minioMetaBucket, fsAppendMetaPath)
+	if err != nil {
+		if err != errFileNotFound {
+			return
+		}
+		fsAppendMeta = fsMeta
+		fsAppendMeta.Parts = nil
+	}
+
+	// Check if a part needs to be appended to
+	part, appendNeeded := partToAppend(fsMeta, fsAppendMeta)
+	if !appendNeeded {
+		return
+	}
+	// Hold write lock on the part so that there is no parallel upload on the part.
+	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(part.Number)))
+	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(part.Number)))
+
+	// Proceed to append "part"
+	fsAppendDataPath := getFSAppendDataPath(uploadID)
+	tmpDataPath := path.Join(tmpMetaPrefix, getUUID())
+	if part.Number != 1 {
+		// Move it to tmp location before appending so that we don't leave inconsitent data
+		// if server crashes during append operation.
+		err = disk.RenameFile(minioMetaBucket, fsAppendDataPath, minioMetaBucket, tmpDataPath)
+		if err != nil {
+			return
+		}
+		// Delete fs-append.json so that we don't leave a stale file if server crashes
+		// when the part is being appended to the tmp file.
+		err = disk.DeleteFile(minioMetaBucket, fsAppendMetaPath)
+		if err != nil {
+			return
+		}
+	}
+	// Path to the part that needs to be appended.
+	partPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, part.Name)
+	offset := int64(0)
+	totalLeft := part.Size
+	buf := make([]byte, readSizeV1)
+	for totalLeft > 0 {
+		curLeft := int64(readSizeV1)
+		if totalLeft < readSizeV1 {
+			curLeft = totalLeft
+		}
+		var n int64
+		n, err = disk.ReadFile(minioMetaBucket, partPath, offset, buf[:curLeft])
+		if n > 0 {
+			if err = disk.AppendFile(minioMetaBucket, tmpDataPath, buf[:n]); err != nil {
+				return
+			}
+		}
+		if err != nil {
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				break
+			}
+			return
+		}
+		offset += n
+		totalLeft -= n
+	}
+	// All good, the part has been appended to the tmp file, rename it back.
+	if err = disk.RenameFile(minioMetaBucket, tmpDataPath, minioMetaBucket, fsAppendDataPath); err != nil {
+		return
+	}
+	fsAppendMeta.AddObjectPart(part.Number, part.Name, part.ETag, part.Size)
+	if err = writeFSMetadata(disk, minioMetaBucket, fsAppendMetaPath, fsAppendMeta); err != nil {
+		return
+	}
+	// If there are more parts that need to be appended to fsAppendDataFile
+	_, appendNeeded = partToAppend(fsMeta, fsAppendMeta)
+	if appendNeeded {
+		go appendParts(disk, bucket, object, uploadID)
+	}
+}
+
 // PutObjectPart - reads incoming data until EOF for the part file on
 // an ongoing multipart transaction. Internally incoming data is
 // written to '.minio/tmp' location and safely renamed to
@@ -299,12 +412,8 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		return "", InvalidUploadID{UploadID: uploadID}
 	}
 
-	// Hold write lock on the part so that there is no parallel upload on the part.
-	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(partID)))
-	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID, strconv.Itoa(partID)))
-
 	partSuffix := fmt.Sprintf("object%d", partID)
-	tmpPartPath := path.Join(tmpMetaPrefix, uploadID+"-"+partSuffix)
+	tmpPartPath := path.Join(tmpMetaPrefix, uploadID+"."+getUUID()+"."+partSuffix)
 
 	// Initialize md5 writer.
 	md5Writer := md5.New()
@@ -366,9 +475,10 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		return "", InvalidUploadID{UploadID: uploadID}
 	}
 
-	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
+	fsMetaPath := pathJoin(uploadIDPath, fsMetaJSONFile)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsMetaPath)
 	if err != nil {
-		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+		return "", toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
 	fsMeta.AddObjectPart(partID, partSuffix, newMD5Hex, size)
 
@@ -380,18 +490,11 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		}
 		return "", toObjectErr(err, minioMetaBucket, partPath)
 	}
-	uploadIDPath = path.Join(mpartMetaPrefix, bucket, object, uploadID)
-	tempFSMetadataPath := path.Join(tmpMetaPrefix, getUUID()+"-"+fsMetaJSONFile)
-	if err = writeFSMetadata(fs.storage, minioMetaBucket, tempFSMetadataPath, fsMeta); err != nil {
-		return "", toObjectErr(err, minioMetaBucket, tempFSMetadataPath)
-	}
-	err = fs.storage.RenameFile(minioMetaBucket, tempFSMetadataPath, minioMetaBucket, path.Join(uploadIDPath, fsMetaJSONFile))
-	if err != nil {
-		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempFSMetadataPath); dErr != nil {
-			return "", toObjectErr(dErr, minioMetaBucket, tempFSMetadataPath)
-		}
-		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
+
+	if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
+		return "", toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
+	go appendParts(fs.storage, bucket, object, uploadID)
 	return newMD5Hex, nil
 }
 
@@ -401,10 +504,10 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, error) {
 	result := ListPartsInfo{}
 
-	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
-	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
+	fsMetaPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, fsMetaJSONFile)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsMetaPath)
 	if err != nil {
-		return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, uploadIDPath)
+		return ListPartsInfo{}, toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
 	// Only parts with higher part numbers will be listed.
 	partIdx := fsMeta.ObjectPartIndex(partNumberMarker)
@@ -502,18 +605,18 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	// 1) no one aborts this multipart upload
 	// 2) no one does a parallel complete-multipart-upload on this
 	// multipart upload
-	nsMutex.Lock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
-	defer nsMutex.Unlock(minioMetaBucket, pathJoin(mpartMetaPrefix, bucket, object, uploadID))
+	nsMutex.Lock(minioMetaBucket, uploadIDPath)
+	defer nsMutex.Unlock(minioMetaBucket, uploadIDPath)
 
 	if !fs.isUploadIDExists(bucket, object, uploadID) {
 		return "", InvalidUploadID{UploadID: uploadID}
 	}
 
-	// Read saved fs metadata for ongoing multipart.
-	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, uploadIDPath)
-	if err != nil {
-		return "", toObjectErr(err, minioMetaBucket, uploadIDPath)
-	}
+	// fs-append.json path
+	fsAppendMetaPath := getFSAppendMetaPath(uploadID)
+	// Lock fs-append.json so that no parallel appendParts() is being done.
+	nsMutex.Lock(minioMetaBucket, fsAppendMetaPath)
+	defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath)
 
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5, err := completeMultipartMD5(parts...)
@@ -521,66 +624,83 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		return "", err
 	}
 
-	tempObj := path.Join(tmpMetaPrefix, uploadID+"-"+"part.1")
-
-	// Allocate staging buffer.
-	var buf = make([]byte, readSizeV1)
-
-	// Loop through all parts, validate them and then commit to disk.
-	for i, part := range parts {
-		partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
-		if partIdx == -1 {
-			return "", InvalidPart{}
-		}
-		if fsMeta.Parts[partIdx].ETag != part.ETag {
-			return "", BadDigest{}
-		}
-		// All parts except the last part has to be atleast 5MB.
-		if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
-			return "", PartTooSmall{
-				PartNumber: part.PartNumber,
-				PartSize:   fsMeta.Parts[partIdx].Size,
-				PartETag:   part.ETag,
-			}
-		}
-		// Construct part suffix.
-		partSuffix := fmt.Sprintf("object%d", part.PartNumber)
-		multipartPartFile := path.Join(mpartMetaPrefix, bucket, object, uploadID, partSuffix)
-		offset := int64(0)
-		totalLeft := fsMeta.Parts[partIdx].Size
-		for totalLeft > 0 {
-			curLeft := int64(readSizeV1)
-			if totalLeft < readSizeV1 {
-				curLeft = totalLeft
-			}
-			var n int64
-			n, err = fs.storage.ReadFile(minioMetaBucket, multipartPartFile, offset, buf[:curLeft])
-			if n > 0 {
-				if err = fs.storage.AppendFile(minioMetaBucket, tempObj, buf[:n]); err != nil {
-					return "", toObjectErr(err, minioMetaBucket, tempObj)
-				}
-			}
-			if err != nil {
-				if err == io.EOF || err == io.ErrUnexpectedEOF {
-					break
-				}
-				if err == errFileNotFound {
-					return "", InvalidPart{}
-				}
-				return "", toObjectErr(err, minioMetaBucket, multipartPartFile)
-			}
-			offset += n
-			totalLeft -= n
-		}
+	// Read saved fs metadata for ongoing multipart.
+	fsMetaPath := pathJoin(uploadIDPath, fsMetaJSONFile)
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsMetaPath)
+	if err != nil {
+		return "", toObjectErr(err, minioMetaBucket, fsMetaPath)
 	}
 
-	// Rename the file back to original location, if not delete the temporary object.
-	err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
-	if err != nil {
-		if dErr := fs.storage.DeleteFile(minioMetaBucket, tempObj); dErr != nil {
-			return "", toObjectErr(dErr, minioMetaBucket, tempObj)
+	fsAppendMeta, err := readFSMetadata(fs.storage, minioMetaBucket, fsAppendMetaPath)
+	if err == nil && isPartsSame(fsAppendMeta.Parts, parts) {
+		fsAppendDataPath := getFSAppendDataPath(uploadID)
+		if err = fs.storage.RenameFile(minioMetaBucket, fsAppendDataPath, bucket, object); err != nil {
+			return "", toObjectErr(err, minioMetaBucket, fsAppendDataPath)
+		}
+		// Remove the append-file metadata file in tmp location as we no longer need it.
+		fs.storage.DeleteFile(minioMetaBucket, fsAppendMetaPath)
+	} else {
+		tempObj := path.Join(tmpMetaPrefix, uploadID+"-"+"part.1")
+
+		// Allocate staging buffer.
+		var buf = make([]byte, readSizeV1)
+
+		// Loop through all parts, validate them and then commit to disk.
+		for i, part := range parts {
+			partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
+			if partIdx == -1 {
+				return "", InvalidPart{}
+			}
+			if fsMeta.Parts[partIdx].ETag != part.ETag {
+				return "", BadDigest{}
+			}
+			// All parts except the last part has to be atleast 5MB.
+			if (i < len(parts)-1) && !isMinAllowedPartSize(fsMeta.Parts[partIdx].Size) {
+				return "", PartTooSmall{
+					PartNumber: part.PartNumber,
+					PartSize:   fsMeta.Parts[partIdx].Size,
+					PartETag:   part.ETag,
+				}
+			}
+			// Construct part suffix.
+			partSuffix := fmt.Sprintf("object%d", part.PartNumber)
+			multipartPartFile := path.Join(mpartMetaPrefix, bucket, object, uploadID, partSuffix)
+			offset := int64(0)
+			totalLeft := fsMeta.Parts[partIdx].Size
+			for totalLeft > 0 {
+				curLeft := int64(readSizeV1)
+				if totalLeft < readSizeV1 {
+					curLeft = totalLeft
+				}
+				var n int64
+				n, err = fs.storage.ReadFile(minioMetaBucket, multipartPartFile, offset, buf[:curLeft])
+				if n > 0 {
+					if err = fs.storage.AppendFile(minioMetaBucket, tempObj, buf[:n]); err != nil {
+						return "", toObjectErr(err, minioMetaBucket, tempObj)
+					}
+				}
+				if err != nil {
+					if err == io.EOF || err == io.ErrUnexpectedEOF {
+						break
+					}
+					if err == errFileNotFound {
+						return "", InvalidPart{}
+					}
+					return "", toObjectErr(err, minioMetaBucket, multipartPartFile)
+				}
+				offset += n
+				totalLeft -= n
+			}
+		}
+
+		// Rename the file back to original location, if not delete the temporary object.
+		err = fs.storage.RenameFile(minioMetaBucket, tempObj, bucket, object)
+		if err != nil {
+			if dErr := fs.storage.DeleteFile(minioMetaBucket, tempObj); dErr != nil {
+				return "", toObjectErr(dErr, minioMetaBucket, tempObj)
+			}
+			return "", toObjectErr(err, bucket, object)
 		}
-		return "", toObjectErr(err, bucket, object)
 	}
 
 	// No need to save part info, since we have concatenated all parts.
@@ -593,13 +713,8 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		}
 		fsMeta.Meta["md5Sum"] = s3MD5
 
-		// Write the metadata to a temp file and rename it to the actual location.
-		tmpMetaPath := path.Join(tmpMetaPrefix, getUUID())
-		if err = writeFSMetadata(fs.storage, minioMetaBucket, tmpMetaPath, fsMeta); err != nil {
-			return "", toObjectErr(err, bucket, object)
-		}
-		fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
-		if err = fs.storage.RenameFile(minioMetaBucket, tmpMetaPath, minioMetaBucket, fsMetaPath); err != nil {
+		fsMetaPath = path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
+		if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
 			return "", toObjectErr(err, bucket, object)
 		}
 	}
@@ -709,6 +824,11 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 		return InvalidUploadID{UploadID: uploadID}
 	}
 
+	fsAppendMetaPath := getFSAppendMetaPath(uploadID)
+	// Lock fs-append.json so that no parallel appendParts() is being done.
+	nsMutex.Lock(minioMetaBucket, fsAppendMetaPath)
+	defer nsMutex.Unlock(minioMetaBucket, fsAppendMetaPath)
+
 	err := fs.abortMultipartUpload(bucket, object, uploadID)
 	return err
 }

cmd/fs-v1.go

@@ -324,7 +324,7 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
 	if err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
-	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, object))
+	fsMeta, err := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile))
 	if err != nil && err != errFileNotFound {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
@@ -460,13 +460,8 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 		fsMeta := newFSMetaV1()
 		fsMeta.Meta = metadata
 
-		// Write the metadata to a temp file and rename it to the actual location.
-		tmpMetaPath := path.Join(tmpMetaPrefix, getUUID())
 		fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
-		if err = writeFSMetadata(fs.storage, minioMetaBucket, tmpMetaPath, fsMeta); err != nil {
-			return "", toObjectErr(err, bucket, object)
-		}
-		if err = fs.storage.RenameFile(minioMetaBucket, tmpMetaPath, minioMetaBucket, fsMetaPath); err != nil {
+		if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
 			return "", toObjectErr(err, bucket, object)
 		}
 	}
@@ -523,7 +518,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 		if fileInfo, err = fs.storage.StatFile(bucket, entry); err != nil {
 			return
 		}
-		fsMeta, mErr := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, entry))
+		fsMeta, mErr := readFSMetadata(fs.storage, minioMetaBucket, path.Join(bucketMetaPrefix, bucket, entry, fsMetaJSONFile))
 		if mErr != nil && mErr != errFileNotFound {
 			return FileInfo{}, mErr
 		}

cmd/generic-handlers.go

@@ -198,9 +198,10 @@ func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			writeErrorResponse(w, r, apiErr, r.URL.Path)
 			return
 		}
-		// Verify if the request date header is more than 5minutes
-		// late, reject such clients.
-		if time.Now().UTC().Sub(amzDate)/time.Minute > time.Duration(5)*time.Minute {
+		// Verify if the request date header is shifted by less than maxSkewTime parameter in the past
+		// or in the future, reject request otherwise.
+		curTime := time.Now().UTC()
+		if curTime.Sub(amzDate) > maxSkewTime || amzDate.Sub(curTime) > maxSkewTime {
 			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
 			return
 		}

cmd/globals.go

@@ -17,6 +17,8 @@
 package cmd
 
 import (
+	"time"
+
 	"github.com/fatih/color"
 	"github.com/minio/minio/pkg/objcache"
 )
@@ -28,7 +30,7 @@ const (
 
 // minio configuration related constants.
 const (
-	globalMinioConfigVersion = "6"
+	globalMinioConfigVersion = "7"
 	globalMinioConfigDir     = ".minio"
 	globalMinioCertsDir      = "certs"
 	globalMinioCertFile      = "public.crt"
@@ -58,6 +60,11 @@ var (
 	maxFormFieldSize = int64(1024 * 1024)
 )
 
+var (
+	// The maximum allowed difference between the request generation time and the server processing time
+	maxSkewTime = 15 * time.Minute
+)
+
 // global colors.
 var (
 	colorBlue = color.New(color.FgBlue).SprintfFunc()

cmd/main.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/minio/cli"
 	"github.com/minio/mc/pkg/console"
-	"github.com/pkg/profile"
 )
 
 var (
@@ -130,6 +129,7 @@ func registerApp() *cli.App {
 	return app
 }
 
+// Verify main command syntax.
 func checkMainSyntax(c *cli.Context) {
 	configPath, err := getConfigPath()
 	if err != nil {
@@ -140,7 +140,7 @@ func checkMainSyntax(c *cli.Context) {
 	}
 }
 
-// Main - main for minio server.
+// Main main for minio server.
 func Main() {
 	app := registerApp()
 	app.Before = func(c *cli.Context) error {
@@ -176,16 +176,9 @@ func Main() {
 		return nil
 	}
 
-	// Set ``MINIO_PROFILE_DIR`` to the directory where profiling information should be persisted
-	profileDir := os.Getenv("MINIO_PROFILE_DIR")
-	// Enable profiler if ``MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
-	switch os.Getenv("MINIO_PROFILER") {
-	case "cpu":
-		defer profile.Start(profile.CPUProfile, profile.ProfilePath(profileDir)).Stop()
-	case "mem":
-		defer profile.Start(profile.MemProfile, profile.ProfilePath(profileDir)).Stop()
-	case "block":
-		defer profile.Start(profile.BlockProfile, profile.ProfilePath(profileDir)).Stop()
+	// Start profiler if env is set.
+	if profiler := os.Getenv("MINIO_PROFILER"); profiler != "" {
+		globalProfiler = startProfiler(profiler)
 	}
 
 	// Run the app - exit on error.

cmd/naughty-disk_test.go

@@ -0,0 +1,123 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// naughtyDisk wraps a POSIX disk and returns programmed errors
+// specified by the developer. The purpose is to simulate errors
+// that are hard to simulate in practise like DiskNotFound.
+// Programmed errors are stored in errors field.
+type naughtyDisk struct {
+	// The real disk
+	disk *posix
+	// Programmed errors: API call number => error to return
+	errors map[int]error
+	// The error to return when no error value is programmed
+	defaultErr error
+	// The current API call number
+	callNR int
+}
+
+func newNaughtyDisk(d *posix, errs map[int]error, defaultErr error) *naughtyDisk {
+	return &naughtyDisk{disk: d, errors: errs, defaultErr: defaultErr}
+}
+
+func (d *naughtyDisk) calcError() (err error) {
+	d.callNR++
+	if err, ok := d.errors[d.callNR]; ok {
+		return err
+	}
+	if d.defaultErr != nil {
+		return d.defaultErr
+	}
+	return nil
+}
+
+func (d *naughtyDisk) MakeVol(volume string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.MakeVol(volume)
+}
+
+func (d *naughtyDisk) ListVols() (vols []VolInfo, err error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.ListVols()
+}
+
+func (d *naughtyDisk) StatVol(volume string) (volInfo VolInfo, err error) {
+	if err := d.calcError(); err != nil {
+		return VolInfo{}, err
+	}
+	return d.disk.StatVol(volume)
+}
+func (d *naughtyDisk) DeleteVol(volume string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.DeleteVol(volume)
+}
+
+func (d *naughtyDisk) ListDir(volume, path string) (entries []string, err error) {
+	if err := d.calcError(); err != nil {
+		return []string{}, err
+	}
+	return d.disk.ListDir(volume, path)
+}
+
+func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
+	if err := d.calcError(); err != nil {
+		return 0, err
+	}
+	return d.disk.ReadFile(volume, path, offset, buf)
+}
+
+func (d *naughtyDisk) AppendFile(volume, path string, buf []byte) error {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.AppendFile(volume, path, buf)
+}
+
+func (d *naughtyDisk) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.RenameFile(srcVolume, srcPath, dstVolume, dstPath)
+}
+
+func (d *naughtyDisk) StatFile(volume string, path string) (file FileInfo, err error) {
+	if err := d.calcError(); err != nil {
+		return FileInfo{}, err
+	}
+	return d.disk.StatFile(volume, path)
+}
+
+func (d *naughtyDisk) DeleteFile(volume string, path string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.DeleteFile(volume, path)
+}
+
+func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.ReadAll(volume, path)
+}

cmd/notifiers.go

@@ -114,7 +114,7 @@ func isElasticQueue(sqsArn arnSQS) bool {
 // Match function matches wild cards in 'pattern' for events.
 func eventMatch(eventType string, events []string) (ok bool) {
 	for _, event := range events {
-		ok = wildcard.Match(event, eventType)
+		ok = wildcard.MatchSimple(event, eventType)
 		if ok {
 			break
 		}

cmd/object-handlers.go

@@ -268,9 +268,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	// Skip the first element if it is '/', split the rest.
-	if strings.HasPrefix(objectSource, "/") {
-		objectSource = objectSource[1:]
-	}
+	objectSource = strings.TrimPrefix(objectSource, "/")
 	splits := strings.SplitN(objectSource, "/", 2)
 
 	// Save sourceBucket and sourceObject extracted from url Path.
@@ -336,10 +334,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	// Create the object.
 	md5Sum, err := api.ObjectAPI.PutObject(bucket, object, size, pipeReader, metadata)
 	if err != nil {
+		// Close the this end of the pipe upon error in PutObject.
+		pipeReader.CloseWithError(err)
 		errorIf(err, "Unable to create an object.")
 		writeErrorResponse(w, r, toAPIErrorCode(err), r.URL.Path)
 		return
 	}
+	// Explicitly close the reader, before fetching object info.
+	pipeReader.Close()
 
 	objInfo, err = api.ObjectAPI.GetObjectInfo(bucket, object)
 	if err != nil {
@@ -354,10 +356,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	setCommonHeaders(w)
 	// write success response.
 	writeSuccessResponse(w, encodedSuccessResponse)
-	// Explicitly close the reader, to avoid fd leaks.
-	pipeReader.Close()
 
-	if eventN.IsBucketNotificationSet(bucket) {
+	if globalEventNotifier.IsBucketNotificationSet(bucket) {
 		// Notify object created event.
 		eventNotify(eventData{
 			Type:    ObjectCreatedCopy,
@@ -456,14 +456,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 	writeSuccessResponse(w, nil)
 
-	// Fetch object info for notifications.
-	objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
-	if err != nil {
-		errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
-		return
-	}
+	if globalEventNotifier.IsBucketNotificationSet(bucket) {
+		// Fetch object info for notifications.
+		objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
+		if err != nil {
+			errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
+			return
+		}
 
-	if eventN.IsBucketNotificationSet(bucket) {
 		// Notify object created event.
 		eventNotify(eventData{
 			Type:    ObjectCreatedPut,
@@ -797,14 +797,14 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	w.Write(encodedSuccessResponse)
 	w.(http.Flusher).Flush()
 
-	// Fetch object info for notifications.
-	objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
-	if err != nil {
-		errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
-		return
-	}
+	if globalEventNotifier.IsBucketNotificationSet(bucket) {
+		// Fetch object info for notifications.
+		objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
+		if err != nil {
+			errorIf(err, "Unable to fetch object info for \"%s\"", path.Join(bucket, object))
+			return
+		}
 
-	if eventN.IsBucketNotificationSet(bucket) {
 		// Notify object created event.
 		eventNotify(eventData{
 			Type:    ObjectCreatedCompleteMultipartUpload,
@@ -851,7 +851,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 	}
 	writeSuccessNoContent(w)
 
-	if eventN.IsBucketNotificationSet(bucket) {
+	if globalEventNotifier.IsBucketNotificationSet(bucket) {
 		// Notify object deleted event.
 		eventNotify(eventData{
 			Type:   ObjectRemovedDelete,

cmd/posix.go

@@ -107,10 +107,8 @@ func newPosix(diskPath string) (StorageAPI, error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			// Disk not found create it.
-			if err = os.MkdirAll(diskPath, 0777); err != nil {
-				return fs, err
-			}
-			return fs, nil
+			err = os.MkdirAll(diskPath, 0777)
+			return fs, err
 		}
 		return fs, err
 	}

cmd/routers.go

@@ -69,6 +69,11 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler {
 		ObjectAPI: objAPI,
 	}
 
+	// Initialize Controller.
+	ctrlHandlers := &controllerAPIHandlers{
+		ObjectAPI: objAPI,
+	}
+
 	// Initialize and monitor shutdown signals.
 	err = initGracefulShutdown(os.Exit)
 	fatalIf(err, "Unable to initialize graceful shutdown operation")
@@ -98,7 +103,7 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler {
 	// FIXME: till net/rpc auth is brought in "minio control" can be enabled only though
 	// this env variable.
 	if os.Getenv("MINIO_CONTROL") != "" {
-		registerControlRPCRouter(mux, objAPI)
+		registerControlRPCRouter(mux, ctrlHandlers)
 	}
 
 	// set environmental variable MINIO_BROWSER=off to disable minio web browser.

cmd/rpc-server.go

@@ -16,11 +16,7 @@ type storageServer struct {
 
 // MakeVolHandler - make vol handler is rpc wrapper for MakeVol operation.
 func (s *storageServer) MakeVolHandler(arg *string, reply *GenericReply) error {
-	err := s.storage.MakeVol(*arg)
-	if err != nil {
-		return err
-	}
-	return nil
+	return s.storage.MakeVol(*arg)
 }
 
 // ListVolsHandler - list vols handler is rpc wrapper for ListVols operation.
@@ -46,11 +42,7 @@ func (s *storageServer) StatVolHandler(arg *string, reply *VolInfo) error {
 // DeleteVolHandler - delete vol handler is a rpc wrapper for
 // DeleteVol operation.
 func (s *storageServer) DeleteVolHandler(arg *string, reply *GenericReply) error {
-	err := s.storage.DeleteVol(*arg)
-	if err != nil {
-		return err
-	}
-	return nil
+	return s.storage.DeleteVol(*arg)
 }
 
 /// File operations

cmd/server-main.go

@@ -251,20 +251,20 @@ func serverMain(c *cli.Context) {
 		ignoredDisks: ignoredDisks,
 	})
 
-	apiServer := NewMuxServer(serverAddress, handler)
+	apiServer := NewServerMux(serverAddress, handler)
 
 	// Fetch endpoints which we are going to serve from.
 	endPoints := finalizeEndpoints(tls, &apiServer.Server)
 
-	// Prints the formatted startup message.
-	printStartupMessage(endPoints)
-
 	// Register generic callbacks.
 	globalShutdownCBs.AddGenericCB(func() errCode {
 		// apiServer.Stop()
 		return exitSuccess
 	})
 
+	// Prints the formatted startup message.
+	printStartupMessage(endPoints)
+
 	// Start server.
 	// Configure TLS if certs are available.
 	if tls {

cmd/server-mux.go

@@ -50,28 +50,44 @@ type ConnBuf struct {
 	offset int
 }
 
-// MuxConn - implements a Read() which streams twice the firs bytes from
+// ConnMux - implements a Read() which streams twice the firs bytes from
 // the incoming connection, to help peeking protocol
-type MuxConn struct {
+type ConnMux struct {
 	net.Conn
 	lastError error
 	dataBuf   ConnBuf
 }
 
-// NewMuxConn - creates a new MuxConn instance
-func NewMuxConn(c net.Conn) *MuxConn {
+func longestWord(strings []string) int {
+	maxLen := 0
+	for _, m := range defaultHTTP1Methods {
+		if maxLen < len(m) {
+			maxLen = len(m)
+		}
+	}
+	for _, m := range defaultHTTP2Methods {
+		if maxLen < len(m) {
+			maxLen = len(m)
+		}
+	}
+
+	return maxLen
+}
+
+// NewConnMux - creates a new ConnMux instance
+func NewConnMux(c net.Conn) *ConnMux {
 	h1 := longestWord(defaultHTTP1Methods)
 	h2 := longestWord(defaultHTTP2Methods)
 	max := h1
 	if h2 > max {
 		max = h2
 	}
-	return &MuxConn{Conn: c, dataBuf: ConnBuf{buffer: make([]byte, max+1)}}
+	return &ConnMux{Conn: c, dataBuf: ConnBuf{buffer: make([]byte, max+1)}}
 }
 
 // PeekProtocol - reads the first bytes, then checks if it is similar
 // to one of the default http methods
-func (c *MuxConn) PeekProtocol() string {
+func (c *ConnMux) PeekProtocol() string {
 	var n int
 	n, c.lastError = c.Conn.Read(c.dataBuf.buffer)
 	if n == 0 || (c.lastError != nil && c.lastError != io.EOF) {
@@ -91,9 +107,9 @@ func (c *MuxConn) PeekProtocol() string {
 	return "tls"
 }
 
-// Read - streams the MuxConn buffer when reset flag is activated, otherwise
+// Read - streams the ConnMux buffer when reset flag is activated, otherwise
 // streams from the incoming network connection
-func (c *MuxConn) Read(b []byte) (int, error) {
+func (c *ConnMux) Read(b []byte) (int, error) {
 	if c.dataBuf.unRead {
 		n := copy(b, c.dataBuf.buffer[c.dataBuf.offset:])
 		c.dataBuf.offset += n
@@ -118,67 +134,40 @@ func (c *MuxConn) Read(b []byte) (int, error) {
 	return c.Conn.Read(b)
 }
 
-// MuxListener - encapuslates the standard net.Listener to inspect
+// ListenerMux - encapuslates the standard net.Listener to inspect
 // the communication protocol upon network connection
-type MuxListener struct {
+type ListenerMux struct {
 	net.Listener
 	config *tls.Config
-	wg     *sync.WaitGroup
-}
-
-// NewMuxListener - creates new MuxListener, returns error when cert/key files are not found
-// or invalid
-func NewMuxListener(listener net.Listener, wg *sync.WaitGroup, certPath, keyPath string) (*MuxListener, error) {
-	var err error
-	var config *tls.Config
-	config = nil
-
-	if certPath != "" {
-		config = &tls.Config{}
-		if config.NextProtos == nil {
-			config.NextProtos = []string{"http/1.1", "h2"}
-		}
-		config.Certificates = make([]tls.Certificate, 1)
-		config.Certificates[0], err = tls.LoadX509KeyPair(certPath, keyPath)
-		if err != nil {
-			return &MuxListener{}, err
-		}
-	}
-
-	l := &MuxListener{Listener: listener, config: config, wg: wg}
-
-	return l, nil
 }
 
 // Accept - peek the protocol to decide if we should wrap the
 // network stream with the TLS server
-func (l *MuxListener) Accept() (net.Conn, error) {
-	c, err := l.Listener.Accept()
+func (l *ListenerMux) Accept() (net.Conn, error) {
+	conn, err := l.Listener.Accept()
 	if err != nil {
-		return c, err
+		return conn, err
 	}
-
-	cmux := NewMuxConn(c)
-	protocol := cmux.PeekProtocol()
+	connMux := NewConnMux(conn)
+	protocol := connMux.PeekProtocol()
 	if protocol == "tls" {
-		return tls.Server(cmux, l.config), nil
+		return tls.Server(connMux, l.config), nil
 	}
-	return cmux, nil
+	return connMux, nil
 }
 
 // Close Listener
-func (l *MuxListener) Close() error {
+func (l *ListenerMux) Close() error {
 	if l == nil {
 		return nil
 	}
-
 	return l.Listener.Close()
 }
 
-// MuxServer - the main mux server
-type MuxServer struct {
+// ServerMux - the main mux server
+type ServerMux struct {
 	http.Server
-	listener        *MuxListener
+	listener        *ListenerMux
 	WaitGroup       *sync.WaitGroup
 	GracefulTimeout time.Duration
 	mu              sync.Mutex // guards closed, conns, and listener
@@ -186,14 +175,14 @@ type MuxServer struct {
 	conns           map[net.Conn]http.ConnState // except terminal states
 }
 
-// NewMuxServer constructor to create a MuxServer
-func NewMuxServer(addr string, handler http.Handler) *MuxServer {
-	m := &MuxServer{
+// NewServerMux constructor to create a ServerMux
+func NewServerMux(addr string, handler http.Handler) *ServerMux {
+	m := &ServerMux{
 		Server: http.Server{
 			Addr: addr,
-			// Adding timeout of 10 minutes for unresponsive client connections.
-			ReadTimeout:    10 * time.Minute,
-			WriteTimeout:   10 * time.Minute,
+			// Do not add any timeouts Golang net.Conn
+			// closes connections right after 10mins even
+			// if they are not idle.
 			Handler:        handler,
 			MaxHeaderBytes: 1 << 20,
 		},
@@ -211,23 +200,31 @@ func NewMuxServer(addr string, handler http.Handler) *MuxServer {
 // ListenAndServeTLS - similar to the http.Server version. However, it has the
 // ability to redirect http requests to the correct HTTPS url if the client
 // mistakenly initiates a http connection over the https port
-func (m *MuxServer) ListenAndServeTLS(certFile, keyFile string) error {
+func (m *ServerMux) ListenAndServeTLS(certFile, keyFile string) error {
 	listener, err := net.Listen("tcp", m.Server.Addr)
 	if err != nil {
 		return err
 	}
-	mux, err := NewMuxListener(listener, m.WaitGroup, mustGetCertFile(), mustGetKeyFile())
+
+	config := &tls.Config{} // Always instantiate.
+	if config.NextProtos == nil {
+		config.NextProtos = []string{"http/1.1", "h2"}
+	}
+	config.Certificates = make([]tls.Certificate, 1)
+	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		return err
 	}
 
+	listenerMux := &ListenerMux{Listener: listener, config: config}
+
 	m.mu.Lock()
-	m.listener = mux
+	m.listener = listenerMux
 	m.mu.Unlock()
 
-	err = http.Serve(mux,
+	err = http.Serve(listenerMux,
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// We reach here when MuxListener.MuxConn is not wrapped with tls.Server
+			// We reach here when ListenerMux.ConnMux is not wrapped with tls.Server
 			if r.TLS == nil {
 				u := url.URL{
 					Scheme:   "https",
@@ -248,42 +245,23 @@ func (m *MuxServer) ListenAndServeTLS(certFile, keyFile string) error {
 }
 
 // ListenAndServe - Same as the http.Server version
-func (m *MuxServer) ListenAndServe() error {
+func (m *ServerMux) ListenAndServe() error {
 	listener, err := net.Listen("tcp", m.Server.Addr)
 	if err != nil {
 		return err
 	}
 
-	mux, err := NewMuxListener(listener, m.WaitGroup, "", "")
-	if err != nil {
-		return err
-	}
+	listenerMux := &ListenerMux{Listener: listener, config: &tls.Config{}}
 
 	m.mu.Lock()
-	m.listener = mux
+	m.listener = listenerMux
 	m.mu.Unlock()
 
-	return m.Server.Serve(mux)
-}
-
-func longestWord(strings []string) int {
-	maxLen := 0
-	for _, m := range defaultHTTP1Methods {
-		if maxLen < len(m) {
-			maxLen = len(m)
-		}
-	}
-	for _, m := range defaultHTTP2Methods {
-		if maxLen < len(m) {
-			maxLen = len(m)
-		}
-	}
-
-	return maxLen
+	return m.Server.Serve(listenerMux)
 }
 
 // Close initiates the graceful shutdown
-func (m *MuxServer) Close() error {
+func (m *ServerMux) Close() error {
 	m.mu.Lock()
 	if m.closed {
 		return errors.New("Server has been closed")
@@ -320,7 +298,7 @@ func (m *MuxServer) Close() error {
 }
 
 // connState setups the ConnState tracking hook to know which connections are idle
-func (m *MuxServer) connState() {
+func (m *ServerMux) connState() {
 	// Set our ConnState to track idle connections
 	m.Server.ConnState = func(c net.Conn, cs http.ConnState) {
 		m.mu.Lock()
@@ -358,7 +336,7 @@ func (m *MuxServer) connState() {
 }
 
 // forgetConn removes c from conns and decrements WaitGroup
-func (m *MuxServer) forgetConn(c net.Conn) {
+func (m *ServerMux) forgetConn(c net.Conn) {
 	if _, ok := m.conns[c]; ok {
 		delete(m.conns, c)
 		m.WaitGroup.Done()

cmd/server-mux_test.go

@@ -40,19 +40,19 @@ import (
 
 func TestClose(t *testing.T) {
 	// Create ServerMux
-	m := NewMuxServer("", nil)
+	m := NewServerMux("", nil)
 
 	if err := m.Close(); err != nil {
 		t.Error("Server errored while trying to Close", err)
 	}
 }
 
-func TestMuxServer(t *testing.T) {
+func TestServerMux(t *testing.T) {
 	ts := httptest.NewUnstartedServer(nil)
 	defer ts.Close()
 
 	// Create ServerMux
-	m := NewMuxServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	m := NewServerMux("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, "hello")
 	}))
 
@@ -60,12 +60,12 @@ func TestMuxServer(t *testing.T) {
 	ts.Config = &m.Server
 	ts.Start()
 
-	// Create a MuxListener
-	ml, err := NewMuxListener(ts.Listener, m.WaitGroup, "", "")
-	if err != nil {
-		t.Fatal(err)
+	// Create a ListenerMux
+	lm := &ListenerMux{
+		Listener: ts.Listener,
+		config:   &tls.Config{},
 	}
-	m.listener = ml
+	m.listener = lm
 
 	client := http.Client{}
 	res, err := client.Get(ts.URL)
@@ -105,7 +105,7 @@ func TestServerCloseBlocking(t *testing.T) {
 	defer ts.Close()
 
 	// Create ServerMux
-	m := NewMuxServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	m := NewServerMux("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, "hello")
 	}))
 
@@ -113,18 +113,17 @@ func TestServerCloseBlocking(t *testing.T) {
 	ts.Config = &m.Server
 	ts.Start()
 
-	// Create a MuxListener
-	// var err error
-	ml, err := NewMuxListener(ts.Listener, m.WaitGroup, "", "")
-	if err != nil {
-		t.Fatal(err)
+	// Create a ListenerMux.
+	lm := &ListenerMux{
+		Listener: ts.Listener,
+		config:   &tls.Config{},
 	}
-	m.listener = ml
+	m.listener = lm
 
 	dial := func() net.Conn {
 		c, cerr := net.Dial("tcp", ts.Listener.Addr().String())
 		if cerr != nil {
-			t.Fatal(err)
+			t.Fatal(cerr)
 		}
 		return c
 	}
@@ -137,7 +136,7 @@ func TestServerCloseBlocking(t *testing.T) {
 	cidle := dial()
 	defer cidle.Close()
 	cidle.Write([]byte("HEAD / HTTP/1.1\r\nHost: foo\r\n\r\n"))
-	_, err = http.ReadResponse(bufio.NewReader(cidle), nil)
+	_, err := http.ReadResponse(bufio.NewReader(cidle), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,14 +152,14 @@ func TestServerCloseBlocking(t *testing.T) {
 	m.mu.Unlock()
 }
 
-func TestListenAndServe(t *testing.T) {
+func TestListenAndServePlain(t *testing.T) {
 	wait := make(chan struct{})
 	addr := "127.0.0.1:" + strconv.Itoa(getFreePort())
 	errc := make(chan error)
 	once := &sync.Once{}
 
 	// Create ServerMux and when we receive a request we stop waiting
-	m := NewMuxServer(addr, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	m := NewServerMux(addr, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, "hello")
 		once.Do(func() { close(wait) })
 	}))
@@ -171,6 +170,8 @@ func TestListenAndServe(t *testing.T) {
 	// Make sure we don't block by closing wait after a timeout
 	tf := time.AfterFunc(time.Millisecond*500, func() { errc <- errors.New("Unable to connect to server") })
 
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
 	// Keep trying the server until it's accepting connections
 	go func() {
 		client := http.Client{Timeout: time.Millisecond * 10}
@@ -182,9 +183,12 @@ func TestListenAndServe(t *testing.T) {
 			}
 		}
 
+		wg.Done()
 		tf.Stop() // Cancel the timeout since we made a successful request
 	}()
 
+	wg.Wait()
+
 	// Block until we get an error or wait closed
 	select {
 	case err := <-errc:
@@ -204,7 +208,7 @@ func TestListenAndServeTLS(t *testing.T) {
 	once := &sync.Once{}
 
 	// Create ServerMux and when we receive a request we stop waiting
-	m := NewMuxServer(addr, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	m := NewServerMux(addr, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, "hello")
 		once.Do(func() { close(wait) })
 	}))
@@ -230,6 +234,8 @@ func TestListenAndServeTLS(t *testing.T) {
 
 	// Make sure we don't block by closing wait after a timeout
 	tf := time.AfterFunc(time.Millisecond*500, func() { errc <- errors.New("Unable to connect to server") })
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
 
 	// Keep trying the server until it's accepting connections
 	go func() {
@@ -248,9 +254,12 @@ func TestListenAndServeTLS(t *testing.T) {
 			}
 		}
 
+		wg.Done()
 		tf.Stop() // Cancel the timeout since we made a successful request
 	}()
 
+	wg.Wait()
+
 	// Block until we get an error or wait closed
 	select {
 	case err := <-errc:

cmd/server-rlimit-nix.go

@@ -37,15 +37,7 @@ func setMaxOpenFiles() error {
 	// TO increase this limit further user has to manually edit
 	// `/etc/security/limits.conf`
 	rLimit.Cur = rLimit.Max
-	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
-	if err != nil {
-		return err
-	}
-	err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
-	if err != nil {
-		return err
-	}
-	return nil
+	return syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
 }
 
 // Set max memory used by minio as a process, this value is usually

cmd/server-startup-msg.go

@@ -60,6 +60,11 @@ func printServerCommonMsg(endPoints []string) {
 	console.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKeyID)))
 	console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretAccessKey)))
 	console.Println(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
+	arnMsg := colorBlue("SqsARNs: ")
+	for queueArn := range globalEventNotifier.queueTargets {
+		arnMsg += colorBold(fmt.Sprintf(getFormatStr(len(queueArn), 2), queueArn))
+	}
+	console.Println(arnMsg)
 
 	console.Println(colorBlue("\nBrowser Access:"))
 	console.Println(fmt.Sprintf(getFormatStr(len(endPointStr), 3), endPointStr))

cmd/server_test.go

@@ -75,10 +75,9 @@ func (s *TestSuiteCommon) TestAuth(c *C) {
 	c.Assert(len(accessID), Equals, minioAccessID)
 }
 
-// TestBucketNotification - Inserts the bucket notification and
-// verifies it by fetching the notification back.
+// TestBucketNotification - Inserts the bucket notification and verifies it by fetching the notification back.
 func (s *TestSuiteCommon) TestBucketNotification(c *C) {
-	// Sample bucket notification
+	// Sample bucket notification.
 	bucketNotificationBuf := `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`
 
 	// generate a random bucket Name.
@@ -100,7 +99,7 @@ func (s *TestSuiteCommon) TestBucketNotification(c *C) {
 	c.Assert(err, IsNil)
 
 	client = http.Client{}
-	// execute the HTTP request to create bucket.
+	// execute the HTTP request.
 	response, err = client.Do(request)
 
 	c.Assert(err, IsNil)
@@ -128,7 +127,7 @@ func (s *TestSuiteCommon) TestBucketNotification(c *C) {
 	c.Assert(err, IsNil)
 
 	client = http.Client{}
-	// execute the HTTP request to create bucket.
+	// execute the HTTP request.
 	response, err = client.Do(request)
 	c.Assert(err, IsNil)
 
@@ -140,7 +139,7 @@ func (s *TestSuiteCommon) TestBucketNotification(c *C) {
 	c.Assert(err, IsNil)
 
 	client = http.Client{}
-	// execute the HTTP request to create bucket.
+	// execute the HTTP request.
 	response, err = client.Do(request)
 	c.Assert(err, IsNil)
 
@@ -152,10 +151,21 @@ func (s *TestSuiteCommon) TestBucketNotification(c *C) {
 	c.Assert(err, IsNil)
 
 	client = http.Client{}
-	// execute the HTTP request to create bucket.
+	// execute the HTTP request.
 	response, err = client.Do(request)
 	c.Assert(err, IsNil)
 	verifyError(c, response, "InvalidArgument", "A specified event is not supported for notifications.", http.StatusBadRequest)
+
+	bucketNotificationDuplicates := `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`
+	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
+		int64(len(bucketNotificationDuplicates)), bytes.NewReader([]byte(bucketNotificationDuplicates)), s.accessKey, s.secretKey)
+	c.Assert(err, IsNil)
+
+	client = http.Client{}
+	// execute the HTTP request.
+	response, err = client.Do(request)
+	c.Assert(err, IsNil)
+	verifyError(c, response, "InvalidArgument", "Configurations overlap. Configurations on the same bucket cannot share a common event type.", http.StatusBadRequest)
 }
 
 // TestBucketPolicy - Inserts the bucket policy and verifies it by fetching the policy back.
@@ -299,6 +309,76 @@ func (s *TestSuiteCommon) TestDeleteBucketNotEmpty(c *C) {
 
 }
 
+func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *C) {
+	// generate a random bucket name.
+	bucketName := getRandomBucketName()
+	// HTTP request to create the bucket.
+	request, err := newTestSignedRequest("PUT", getMakeBucketURL(s.endPoint, bucketName),
+		0, nil, s.accessKey, s.secretKey)
+	c.Assert(err, IsNil)
+
+	client := http.Client{}
+	// execute the request.
+	response, err := client.Do(request)
+	c.Assert(err, IsNil)
+	// assert the http response status code.
+	c.Assert(response.StatusCode, Equals, http.StatusOK)
+
+	objectName := "prefix/myobject"
+	delObjReq := DeleteObjectsRequest{
+		Quiet: false,
+	}
+	for i := 0; i < 10; i++ {
+		// Obtain http request to upload object.
+		// object Name contains a prefix.
+		objName := fmt.Sprintf("%d/%s", i, objectName)
+		request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, objName),
+			0, nil, s.accessKey, s.secretKey)
+		c.Assert(err, IsNil)
+
+		client = http.Client{}
+		// execute the http request.
+		response, err = client.Do(request)
+		c.Assert(err, IsNil)
+		// assert the status of http response.
+		c.Assert(response.StatusCode, Equals, http.StatusOK)
+		// Append all objects.
+		delObjReq.Objects = append(delObjReq.Objects, ObjectIdentifier{
+			ObjectName: objName,
+		})
+	}
+	// Append a non-existent object for which the response should be marked
+	// as deleted.
+	delObjReq.Objects = append(delObjReq.Objects, ObjectIdentifier{
+		ObjectName: fmt.Sprintf("%d/%s", 10, objectName),
+	})
+
+	// Marshal delete request.
+	deleteReqBytes, err := xml.Marshal(delObjReq)
+	c.Assert(err, IsNil)
+
+	// object name was "prefix/myobject", an attempt to delelte "prefix"
+	// Should not delete "prefix/myobject"
+	request, err = newTestSignedRequest("POST", getMultiDeleteObjectURL(s.endPoint, bucketName),
+		int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey)
+	c.Assert(err, IsNil)
+	client = http.Client{}
+	response, err = client.Do(request)
+	c.Assert(err, IsNil)
+	c.Assert(response.StatusCode, Equals, http.StatusOK)
+
+	var deleteResp = DeleteObjectsResponse{}
+	delRespBytes, err := ioutil.ReadAll(response.Body)
+	c.Assert(err, IsNil)
+	err = xml.Unmarshal(delRespBytes, &deleteResp)
+	c.Assert(err, IsNil)
+	for i := 0; i <= 10; i++ {
+		// All the objects should be under deleted list (including non-existent object)
+		c.Assert(deleteResp.DeletedObjects[i], DeepEquals, delObjReq.Objects[i])
+	}
+	c.Assert(len(deleteResp.Errors), Equals, 0)
+}
+
 // Tests delete object responses and success.
 func (s *TestSuiteCommon) TestDeleteObject(c *C) {
 	// generate a random bucket name.
@@ -1287,7 +1367,7 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *C) {
 	c.Assert(strings.Contains(string(getContent), "<Key>bar</Key>"), Equals, true)
 
 	// create listObjectsV2 request with valid parameters
-	request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "1000"),
+	request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "1000", ""),
 		0, nil, s.accessKey, s.secretKey)
 	c.Assert(err, IsNil)
 	client = http.Client{}
@@ -1298,6 +1378,22 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *C) {
 
 	getContent, err = ioutil.ReadAll(response.Body)
 	c.Assert(strings.Contains(string(getContent), "<Key>bar</Key>"), Equals, true)
+	c.Assert(strings.Contains(string(getContent), "<Owner><ID></ID><DisplayName></DisplayName></Owner>"), Equals, true)
+
+	// create listObjectsV2 request with valid parameters and fetch-owner activated
+	request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "1000", "true"),
+		0, nil, s.accessKey, s.secretKey)
+	c.Assert(err, IsNil)
+	client = http.Client{}
+	// execute the HTTP request.
+	response, err = client.Do(request)
+	c.Assert(err, IsNil)
+	c.Assert(response.StatusCode, Equals, http.StatusOK)
+
+	getContent, err = ioutil.ReadAll(response.Body)
+	c.Assert(strings.Contains(string(getContent), "<Key>bar</Key>"), Equals, true)
+	c.Assert(strings.Contains(string(getContent), "<Owner><ID>minio</ID><DisplayName>minio</DisplayName></Owner>"), Equals, true)
+
 }
 
 // TestListObjectsHandlerErrors - Setting invalid parameters to List Objects
@@ -1328,7 +1424,7 @@ func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *C) {
 	verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647", http.StatusBadRequest)
 
 	// create listObjectsV2 request with invalid value of max-keys parameter. max-keys is set to -2.
-	request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "-2"),
+	request, err = newTestSignedRequest("GET", getListObjectsV2URL(s.endPoint, bucketName, "-2", ""),
 		0, nil, s.accessKey, s.secretKey)
 	c.Assert(err, IsNil)
 	client = http.Client{}

cmd/signature-v4.go

@@ -246,6 +246,10 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, validate
 
 	query.Set("X-Amz-Algorithm", signV4Algorithm)
 
+	if pSignValues.Date.After(time.Now().UTC()) {
+		return ErrRequestNotReadyYet
+	}
+
 	if time.Now().UTC().Sub(pSignValues.Date) > time.Duration(pSignValues.Expires) {
 		return ErrExpiredPresignRequest
 	}
@@ -342,8 +346,21 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, validateRegion bo
 		return ErrContentSHA256Mismatch
 	}
 
+	header := req.Header
+
+	// Signature-V4 spec excludes Content-Length from signed headers list for signature calculation.
+	// But some clients deviate from this rule. Hence we consider Content-Length for signature
+	// calculation to be compatible with such clients.
+	for _, h := range signV4Values.SignedHeaders {
+		if h == "content-length" {
+			header = cloneHeader(req.Header)
+			header.Set("content-length", strconv.FormatInt(r.ContentLength, 10))
+			break
+		}
+	}
+
 	// Extract all the signed headers along with its values.
-	extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, req.Header)
+	extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, header)
 	if errCode != ErrNone {
 		return errCode
 	}
@@ -361,9 +378,8 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, validateRegion bo
 		if !isValidRegion(sRegion, region) {
 			return ErrInvalidRegion
 		}
-	} else {
-		region = sRegion
 	}
+	region = sRegion
 
 	// Extract date, if not present throw error.
 	var date string

cmd/test-utils_test.go

@@ -623,6 +623,13 @@ func getDeleteObjectURL(endPoint, bucketName, objectName string) string {
 	return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{})
 }
 
+// return URL for deleting multiple objects from a bucket.
+func getMultiDeleteObjectURL(endPoint, bucketName string) string {
+	queryValue := url.Values{}
+	queryValue.Set("delete", "")
+	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
+}
+
 // return URL for HEAD on the object.
 func getHeadObjectURL(endPoint, bucketName, objectName string) string {
 	return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{})
@@ -699,12 +706,15 @@ func getListObjectsV1URL(endPoint, bucketName string, maxKeys string) string {
 }
 
 // return URL for listing objects in the bucket with V2 API.
-func getListObjectsV2URL(endPoint, bucketName string, maxKeys string) string {
+func getListObjectsV2URL(endPoint, bucketName string, maxKeys string, fetchOwner string) string {
 	queryValue := url.Values{}
 	queryValue.Set("list-type", "2") // Enables list objects V2 URL.
 	if maxKeys != "" {
 		queryValue.Set("max-keys", maxKeys)
 	}
+	if fetchOwner != "" {
+		queryValue.Set("fetch-owner", fetchOwner)
+	}
 	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
 }
 

cmd/update-main.go

@@ -143,10 +143,10 @@ func getReleaseUpdate(updateURL string, noError bool) updateMessage {
 	switch runtime.GOOS {
 	case "windows":
 		// For windows.
-		downloadURL = newUpdateURLPrefix + "/minio.exe?update=yes"
+		downloadURL = newUpdateURLPrefix + "/minio.exe"
 	default:
 		// For all other operating systems.
-		downloadURL = newUpdateURLPrefix + "/minio?update=yes"
+		downloadURL = newUpdateURLPrefix + "/minio"
 	}
 
 	// Initialize update message.

cmd/utils.go

@@ -19,13 +19,30 @@ package cmd
 import (
 	"encoding/base64"
 	"encoding/xml"
+	"errors"
 	"io"
+	"net/http"
 	"os"
+	"os/exec"
 	"strings"
 	"sync"
 	"syscall"
+
+	"github.com/pkg/profile"
 )
 
+// make a copy of http.Header
+func cloneHeader(h http.Header) http.Header {
+	h2 := make(http.Header, len(h))
+	for k, vv := range h {
+		vv2 := make([]string, len(vv))
+		copy(vv2, vv)
+		h2[k] = vv2
+
+	}
+	return h2
+}
+
 // xmlDecoder provide decoded value in xml.
 func xmlDecoder(body io.Reader, v interface{}, size int64) error {
 	var lbody io.Reader
@@ -145,8 +162,39 @@ func initGracefulShutdown(onExitFn onExitFunc) error {
 	return startMonitorShutdownSignal(onExitFn)
 }
 
+type shutdownSignal int
+
+const (
+	shutdownHalt = iota
+	shutdownRestart
+)
+
+// Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
+func startProfiler(profiler string) interface {
+	Stop()
+} {
+	// Set ``MINIO_PROFILE_DIR`` to the directory where profiling information should be persisted
+	profileDir := os.Getenv("MINIO_PROFILE_DIR")
+	// Enable profiler if ``MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
+	switch profiler {
+	case "cpu":
+		return profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
+	case "mem":
+		return profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
+	case "block":
+		return profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
+	default:
+		return nil
+	}
+}
+
 // Global shutdown signal channel.
-var globalShutdownSignalCh = make(chan struct{}, 1)
+var globalShutdownSignalCh = make(chan shutdownSignal, 1)
+
+// Global profiler to be used by shutdown go-routine.
+var globalProfiler interface {
+	Stop()
+}
 
 // Start to monitor shutdownSignal to execute shutdown callbacks
 func startMonitorShutdownSignal(onExitFn onExitFunc) error {
@@ -154,20 +202,27 @@ func startMonitorShutdownSignal(onExitFn onExitFunc) error {
 	if onExitFn == nil {
 		return errInvalidArgument
 	}
+
+	// Start listening on shutdown signal.
 	go func() {
 		defer close(globalShutdownSignalCh)
+
 		// Monitor signals.
 		trapCh := signalTrap(os.Interrupt, syscall.SIGTERM)
 		for {
 			select {
 			case <-trapCh:
 				// Initiate graceful shutdown.
-				globalShutdownSignalCh <- struct{}{}
-			case <-globalShutdownSignalCh:
+				globalShutdownSignalCh <- shutdownHalt
+			case signal := <-globalShutdownSignalCh:
 				// Call all object storage shutdown callbacks and exit for emergency
 				for _, callback := range globalShutdownCBs.GetObjectLayerCBs() {
 					exitCode := callback()
 					if exitCode != exitSuccess {
+						// If global profiler is set stop before we exit.
+						if globalProfiler != nil {
+							globalProfiler.Stop()
+						}
 						onExitFn(int(exitCode))
 					}
 
@@ -176,9 +231,42 @@ func startMonitorShutdownSignal(onExitFn onExitFunc) error {
 				for _, callback := range globalShutdownCBs.GetGenericCBs() {
 					exitCode := callback()
 					if exitCode != exitSuccess {
+						// If global profiler is set stop before we exit.
+						if globalProfiler != nil {
+							globalProfiler.Stop()
+						}
 						onExitFn(int(exitCode))
 					}
 				}
+				// All shutdown callbacks ensure that the server is safely terminated
+				// and any concurrent process could be started again
+				if signal == shutdownRestart {
+					path := os.Args[0]
+					cmdArgs := os.Args[1:]
+					cmd := exec.Command(path, cmdArgs...)
+					cmd.Stdout = os.Stdout
+					cmd.Stderr = os.Stderr
+
+					err := cmd.Start()
+					if err != nil {
+						errorIf(errors.New("Unable to reboot."), err.Error())
+					}
+
+					// If global profiler is set stop before we exit.
+					if globalProfiler != nil {
+						globalProfiler.Stop()
+					}
+
+					// Successfully forked.
+					onExitFn(int(exitSuccess))
+				}
+
+				// If global profiler is set stop before we exit.
+				if globalProfiler != nil {
+					globalProfiler.Stop()
+				}
+
+				// Exit as success if no errors.
 				onExitFn(int(exitSuccess))
 			}
 		}

cmd/utils_test.go

@@ -16,7 +16,35 @@
 
 package cmd
 
-import "testing"
+import (
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+// Tests http.Header clone.
+func TestCloneHeader(t *testing.T) {
+	headers := []http.Header{
+		{
+			"Content-Type":   {"text/html; charset=UTF-8"},
+			"Content-Length": {"0"},
+		},
+		{
+			"Content-Length": {"0", "1", "2"},
+		},
+		{
+			"Expires":          {"-1"},
+			"Content-Length":   {"0"},
+			"Content-Encoding": {"gzip"},
+		},
+	}
+	for i, header := range headers {
+		clonedHeader := cloneHeader(header)
+		if !reflect.DeepEqual(header, clonedHeader) {
+			t.Errorf("Test %d failed", i+1)
+		}
+	}
+}
 
 // Tests maximum object size.
 func TestMaxObjectSize(t *testing.T) {

cmd/web-handlers.go

@@ -17,7 +17,10 @@
 package cmd
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path"
@@ -30,6 +33,7 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/rpc/v2/json2"
+	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/miniobrowser"
 )
 
@@ -392,7 +396,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if eventN.IsBucketNotificationSet(bucket) {
+	if globalEventNotifier.IsBucketNotificationSet(bucket) {
 		// Notify object created event.
 		eventNotify(eventData{
 			Type:    ObjectCreatedPut,
@@ -482,3 +486,98 @@ func writeWebErrorResponse(w http.ResponseWriter, err error) {
 	w.WriteHeader(apiErr.HTTPStatusCode)
 	w.Write([]byte(apiErr.Description))
 }
+
+// GetBucketPolicyArgs - get bucket policy args.
+type GetBucketPolicyArgs struct {
+	BucketName string `json:"bucketName"`
+	Prefix     string `json:"prefix"`
+}
+
+// GetBucketPolicyRep - get bucket policy reply.
+type GetBucketPolicyRep struct {
+	UIVersion string              `json:"uiVersion"`
+	Policy    policy.BucketPolicy `json:"policy"`
+}
+
+func readBucketAccessPolicy(objAPI ObjectLayer, bucketName string) (policy.BucketAccessPolicy, error) {
+	bucketPolicyReader, err := readBucketPolicyJSON(bucketName, objAPI)
+	if err != nil {
+		if _, ok := err.(BucketPolicyNotFound); ok {
+			return policy.BucketAccessPolicy{}, nil
+		}
+		return policy.BucketAccessPolicy{}, err
+	}
+
+	bucketPolicyBuf, err := ioutil.ReadAll(bucketPolicyReader)
+	if err != nil {
+		return policy.BucketAccessPolicy{}, err
+	}
+
+	policyInfo := policy.BucketAccessPolicy{}
+	err = json.Unmarshal(bucketPolicyBuf, &policyInfo)
+	if err != nil {
+		return policy.BucketAccessPolicy{}, err
+	}
+
+	return policyInfo, nil
+
+}
+
+// GetBucketPolicy - get bucket policy.
+func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolicyArgs, reply *GetBucketPolicyRep) error {
+	if !isJWTReqAuthenticated(r) {
+		return &json2.Error{Message: "Unauthorized request"}
+	}
+
+	policyInfo, err := readBucketAccessPolicy(web.ObjectAPI, args.BucketName)
+	if err != nil {
+		return &json2.Error{Message: err.Error()}
+	}
+
+	bucketPolicy := policy.GetPolicy(policyInfo.Statements, args.BucketName, args.Prefix)
+
+	reply.UIVersion = miniobrowser.UIVersion
+	reply.Policy = bucketPolicy
+
+	return nil
+}
+
+// SetBucketPolicyArgs - set bucket policy args.
+type SetBucketPolicyArgs struct {
+	BucketName string `json:"bucketName"`
+	Prefix     string `json:"prefix"`
+	Policy     string `json:"policy"`
+}
+
+// SetBucketPolicy - set bucket policy.
+func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolicyArgs, reply *WebGenericRep) error {
+	if !isJWTReqAuthenticated(r) {
+		return &json2.Error{Message: "Unauthorized request"}
+	}
+
+	bucketPolicy := policy.BucketPolicy(args.Policy)
+	if !bucketPolicy.IsValidBucketPolicy() {
+		return &json2.Error{Message: "Invalid policy " + args.Policy}
+	}
+
+	policyInfo, err := readBucketAccessPolicy(web.ObjectAPI, args.BucketName)
+	if err != nil {
+		return &json2.Error{Message: err.Error()}
+	}
+
+	policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, bucketPolicy, args.BucketName, args.Prefix)
+
+	data, err := json.Marshal(policyInfo)
+	if err != nil {
+		return &json2.Error{Message: err.Error()}
+	}
+
+	// TODO: update policy statements according to bucket name, prefix and policy arguments.
+	if err := writeBucketPolicy(args.BucketName, web.ObjectAPI, bytes.NewReader(data), int64(len(data))); err != nil {
+		return &json2.Error{Message: err.Error()}
+	}
+
+	reply.UIVersion = miniobrowser.UIVersion
+
+	return nil
+}

cmd/web-handlers_test.go

@@ -24,6 +24,8 @@ import (
 	"net/http/httptest"
 	"strconv"
 	"testing"
+
+	"github.com/minio/minio-go/pkg/policy"
 )
 
 // Authenticate and get JWT token - will be called before every webrpc handler invocation
@@ -711,3 +713,60 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl
 		t.Fatalf("The downloaded file is corrupted")
 	}
 }
+
+// Wrapper for calling GetBucketPolicy Handler
+func TestWebHandlerGetBucketPolicyHandler(t *testing.T) {
+	ExecObjectLayerTest(t, testWebGetBucketPolicyHandler)
+}
+
+// testWebGetBucketPolicyHandler - Test GetBucketPolicy web handler
+func testWebGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	// Register the API end points with XL/FS object layer.
+	apiRouter := initTestWebRPCEndPoint(obj)
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	rootPath, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root folder after the test ends.
+	defer removeAll(rootPath)
+
+	credentials := serverConfig.GetCredential()
+
+	authorization, err := getWebRPCToken(apiRouter, credentials.AccessKeyID, credentials.SecretAccessKey)
+	if err != nil {
+		t.Fatal("Cannot authenticate")
+	}
+
+	rec := httptest.NewRecorder()
+
+	bucketName := getRandomBucketName()
+
+	testCases := []struct {
+		bucketName     string
+		prefix         string
+		expectedResult policy.BucketPolicy
+	}{
+		{bucketName, "", policy.BucketPolicyNone},
+	}
+
+	for i, testCase := range testCases {
+		args := &GetBucketPolicyArgs{BucketName: testCase.bucketName, Prefix: testCase.prefix}
+		reply := &GetBucketPolicyRep{}
+		req, err := newTestWebRPCRequest("Web.GetBucketPolicy", authorization, args)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create HTTP request: <ERROR> %v", i+1, err)
+		}
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			t.Fatalf("Test %d: Expected the response status to be 200, but instead found `%d`", i+1, rec.Code)
+		}
+		if err = getTestWebRPCResponse(rec, &reply); err != nil {
+			t.Fatalf("Test %d: Should succeed but it didn't, %v", i+1, err)
+		}
+		if testCase.expectedResult != reply.Policy {
+			t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, reply.Policy)
+		}
+	}
+}

cmd/xl-v1-errors.go

@@ -22,7 +22,7 @@ import "errors"
 var errXLMaxDisks = errors.New("Number of disks are higher than supported maximum count '16'")
 
 // errXLMinDisks - returned for minimum number of disks.
-var errXLMinDisks = errors.New("Minimum '6' disks are required to enable erasure code")
+var errXLMinDisks = errors.New("Minimum '4' disks are required to enable erasure code")
 
 // errXLNumDisks - returned for odd number of disks.
 var errXLNumDisks = errors.New("Total number of disks should be multiples of '2'")

cmd/xl-v1-list-objects.go

@@ -76,7 +76,7 @@ func (xl xlObjects) listObjects(bucket, prefix, marker, delimiter string, maxKey
 		nextMarker = objInfo.Name
 		objInfos = append(objInfos, objInfo)
 		i++
-		if walkResult.end == true {
+		if walkResult.end {
 			eof = true
 			break
 		}

cmd/xl-v1-metadata.go

@@ -137,11 +137,10 @@ func (m xlMetaV1) IsValid() bool {
 }
 
 // ObjectPartIndex - returns the index of matching object part number.
-func (m xlMetaV1) ObjectPartIndex(partNumber int) (index int) {
+func (m xlMetaV1) ObjectPartIndex(partNumber int) int {
 	for i, part := range m.Parts {
 		if partNumber == part.Number {
-			index = i
-			return index
+			return i
 		}
 	}
 	return -1

cmd/xl-v1-object_test.go

@@ -59,7 +59,6 @@ func TestRepeatPutObjectPart(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-
 }
 
 func TestXLDeleteObjectBasic(t *testing.T) {
@@ -130,7 +129,7 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) {
 	// for a 16 disk setup, quorum is 9. To simulate disks not found yet
 	// quorum is available, we remove disks leaving quorum disks behind.
 	for i := range xl.storageDisks[:7] {
-		xl.storageDisks[i] = newFaultyDisk(xl.storageDisks[i].(*posix), 0)
+		xl.storageDisks[i] = newNaughtyDisk(xl.storageDisks[i].(*posix), nil, errFaultyDisk)
 	}
 	err = obj.DeleteObject(bucket, object)
 	if err != nil {
@@ -144,8 +143,8 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) {
 	}
 
 	// Remove one more disk to 'lose' quorum, by setting it to nil.
-	xl.storageDisks[7] = &faultyDisk{}
-	xl.storageDisks[8] = &faultyDisk{}
+	xl.storageDisks[7] = nil
+	xl.storageDisks[8] = nil
 	err = obj.DeleteObject(bucket, object)
 	if err != toObjectErr(errXLWriteQuorum, bucket, object) {
 		t.Errorf("Expected deleteObject to fail with %v, but failed with %v", toObjectErr(errXLWriteQuorum, bucket, object), err)
@@ -180,15 +179,19 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	xl.objCacheEnabled = false
 	// Make 9 disks offline, which leaves less than quorum number of disks
 	// in a 16 disk XL setup. The original disks are 'replaced' with
-	// faultyDisks that fail after 'f' successful StorageAPI method
+	// naughtyDisks that fail after 'f' successful StorageAPI method
 	// invocations, where f - [0,2)
 	for f := 0; f < 2; f++ {
+		diskErrors := make(map[int]error)
+		for i := 0; i <= f; i++ {
+			diskErrors[i] = nil
+		}
 		for i := range xl.storageDisks[:9] {
 			switch diskType := xl.storageDisks[i].(type) {
 			case *posix:
-				xl.storageDisks[i] = newFaultyDisk(diskType, f)
-			case *faultyDisk:
-				xl.storageDisks[i] = newFaultyDisk(diskType.disk, f)
+				xl.storageDisks[i] = newNaughtyDisk(diskType, diskErrors, errFaultyDisk)
+			case *naughtyDisk:
+				xl.storageDisks[i] = newNaughtyDisk(diskType.disk, diskErrors, errFaultyDisk)
 			}
 		}
 		// Fetch object from store.
@@ -226,15 +229,19 @@ func TestPutObjectNoQuorum(t *testing.T) {
 
 	// Make 9 disks offline, which leaves less than quorum number of disks
 	// in a 16 disk XL setup. The original disks are 'replaced' with
-	// faultyDisks that fail after 'f' successful StorageAPI method
+	// naughtyDisks that fail after 'f' successful StorageAPI method
 	// invocations, where f - [0,3)
 	for f := 0; f < 3; f++ {
+		diskErrors := make(map[int]error)
+		for i := 0; i <= f; i++ {
+			diskErrors[i] = nil
+		}
 		for i := range xl.storageDisks[:9] {
 			switch diskType := xl.storageDisks[i].(type) {
 			case *posix:
-				xl.storageDisks[i] = newFaultyDisk(diskType, f)
-			case *faultyDisk:
-				xl.storageDisks[i] = newFaultyDisk(diskType.disk, f)
+				xl.storageDisks[i] = newNaughtyDisk(diskType, diskErrors, errFaultyDisk)
+			case *naughtyDisk:
+				xl.storageDisks[i] = newNaughtyDisk(diskType.disk, diskErrors, errFaultyDisk)
 			}
 		}
 		// Upload new content to same object "object"

cmd/xl-v1.go

@@ -95,13 +95,7 @@ func checkSufficientDisks(disks []string) error {
 
 // isDiskFound - validates if the disk is found in a list of input disks.
 func isDiskFound(disk string, disks []string) bool {
-	for _, d := range disks {
-		// Disk found return
-		if disk == d {
-			return true
-		}
-	}
-	return false
+	return contains(disks, disk)
 }
 
 // newXLObjects - initialize new xl object layer.
@@ -140,7 +134,7 @@ func newXLObjects(disks, ignoredDisks []string) (ObjectLayer, error) {
 
 	// Initialize meta volume, if volume already exists ignores it.
 	if err := initMetaVolume(storageDisks); err != nil {
-		return nil, fmt.Errorf("Unable to initialize '.minio' meta volume, %s", err)
+		return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
 	}
 
 	// Handles different cases properly.

dist/linux-systemd/readme.md

@@ -26,7 +26,7 @@ EOT
 
 Put minio.service in /etc/systemd/system/
 ```
-curl https://raw.githubusercontent.com/minio/minio/master/dist/linux-systemd/minio.service > /etc/systemd/system/
+( cd /etc/systemd/system/; curl -O https://raw.githubusercontent.com/minio/minio/master/dist/linux-systemd/minio.service )
 ```
 
 Enable startup on boot

docs/erasure/README.md

@@ -26,7 +26,7 @@ Minio's erasure coded backend uses high speed [BLAKE2](https://blog.minio.io/acc
 
 Minio server runs on a variety of hardware, operating systems and virtual/container environments. 
 
-Minio erasure code backend is limited by design to a minimum of 6 drives and a maximum of 16 drives. The hard limit of 16 drives comes from operational experience. Failure domain becomes too large beyond 16 drives. If you need to scale beyond 16 drives, you may run multiple instances of Minio server on different ports. 
+Minio erasure code backend is limited by design to a minimum of 4 drives and a maximum of 16 drives. The hard limit of 16 drives comes from operational experience. Failure domain becomes too large beyond 16 drives. If you need to scale beyond 16 drives, you may run multiple instances of Minio server on different ports. 
 
 #### Reference Physical Hardware: 
 

docs/minio-server-configuration-files-guide.md

@@ -1,4 +1,4 @@
-# Minio Server configuration files Guide [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+# Minio Server Configuration Files Guide [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 In this document we will walk through the configuration files of Minio Server.
 

main.go

@@ -1,4 +1,20 @@
-package main
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main // import "github.com/minio/minio"
 
 import minio "github.com/minio/minio/cmd"
 

pkg/mimedb/db.go

@@ -1840,10 +1840,6 @@ var DB = map[string]struct {
 		ContentType:  "image/vnd.ms-modi",
 		Compressible: false,
 	},
-	"mdp": {
-		ContentType:  "application/dash+xml",
-		Compressible: false,
-	},
 	"me": {
 		ContentType:  "text/troff",
 		Compressible: false,
@@ -2008,6 +2004,10 @@ var DB = map[string]struct {
 		ContentType:  "application/vnd.mophun.certificate",
 		Compressible: false,
 	},
+	"mpd": {
+		ContentType:  "application/dash+xml",
+		Compressible: false,
+	},
 	"mpe": {
 		ContentType:  "video/mpeg",
 		Compressible: false,
@@ -2824,6 +2824,10 @@ var DB = map[string]struct {
 		ContentType:  "application/relax-ng-compact-syntax",
 		Compressible: false,
 	},
+	"rng": {
+		ContentType:  "application/xml",
+		Compressible: false,
+	},
 	"roa": {
 		ContentType:  "application/rpki-roa",
 		Compressible: false,
@@ -3088,6 +3092,14 @@ var DB = map[string]struct {
 		ContentType:  "application/vnd.openxmlformats-officedocument.presentationml.slide",
 		Compressible: false,
 	},
+	"slim": {
+		ContentType:  "text/slim",
+		Compressible: false,
+	},
+	"slm": {
+		ContentType:  "text/slim",
+		Compressible: false,
+	},
 	"slt": {
 		ContentType:  "application/vnd.epson.salt",
 		Compressible: false,

pkg/mimedb/db_test.go

@@ -1,5 +1,5 @@
 /*
- * mime-db: Mime Database, (C) 2015 Minio, Inc.
+ * mime-db: Mime Database, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,24 +14,18 @@
  * limitations under the License.
  */
 
-package mimedb_test
+package mimedb
 
-import (
-	"testing"
+import "testing"
 
-	"github.com/minio/minio/pkg/mimedb"
-
-	. "gopkg.in/check.v1"
-)
-
-func Test(t *testing.T) { TestingT(t) }
-
-type MySuite struct{}
-
-var _ = Suite(&MySuite{})
-
-func (s *MySuite) TestLookup(c *C) {
-	// Test MustLookup.
-	contentType := mimedb.DB["exe"].ContentType
-	c.Assert(contentType, Not(Equals), "")
+func TestMimeLookup(t *testing.T) {
+	// Test mimeLookup.
+	contentType := DB["txt"].ContentType
+	if contentType != "text/plain" {
+		t.Fatalf("Invalid content type are found expected \"application/x-msdownload\", got %s", contentType)
+	}
+	compressible := DB["txt"].Compressible
+	if compressible != false {
+		t.Fatalf("Invalid content type are found expected \"false\", got %t", compressible)
+	}
 }

pkg/wildcard/match.go

@@ -16,123 +16,68 @@
 
 package wildcard
 
-import (
-	"strings"
-	"unicode/utf8"
-)
-
-// Match - finds whether the text matches/satisfies the pattern string.
+// MatchSimple - finds whether the text matches/satisfies the pattern string.
 // supports only '*' wildcard in the pattern.
 // considers a file system path as a flat name space.
-func Match(pattern, text string) bool {
+func MatchSimple(pattern, name string) bool {
 	if pattern == "" {
-		return text == pattern
+		return name == pattern
 	}
 	if pattern == "*" {
 		return true
 	}
-	parts := strings.Split(pattern, "*")
-	if len(parts) == 1 {
-		return text == pattern
+	rname := make([]rune, 0, len(name))
+	rpattern := make([]rune, 0, len(pattern))
+	for _, r := range name {
+		rname = append(rname, r)
 	}
-	tGlob := strings.HasSuffix(pattern, "*")
-	end := len(parts) - 1
-	if !strings.HasPrefix(text, parts[0]) {
-		return false
+	for _, r := range pattern {
+		rpattern = append(rpattern, r)
 	}
-	for i := 1; i < end; i++ {
-		if !strings.Contains(text, parts[i]) {
-			return false
-		}
-		idx := strings.Index(text, parts[i]) + len(parts[i])
-		text = text[idx:]
-	}
-	return tGlob || strings.HasSuffix(text, parts[end])
+	simple := true // Does only wildcard '*' match.
+	return deepMatchRune(rname, rpattern, simple)
 }
 
-// MatchExtended -  finds whether the text matches/satisfies the pattern string.
+// Match -  finds whether the text matches/satisfies the pattern string.
 // supports  '*' and '?' wildcards in the pattern string.
 // unlike path.Match(), considers a path as a flat name space while matching the pattern.
 // The difference is illustrated in the example here https://play.golang.org/p/Ega9qgD4Qz .
-func MatchExtended(pattern, name string) (matched bool) {
-Pattern:
+func Match(pattern, name string) (matched bool) {
+	if pattern == "" {
+		return name == pattern
+	}
+	if pattern == "*" {
+		return true
+	}
+	rname := make([]rune, 0, len(name))
+	rpattern := make([]rune, 0, len(pattern))
+	for _, r := range name {
+		rname = append(rname, r)
+	}
+	for _, r := range pattern {
+		rpattern = append(rpattern, r)
+	}
+	simple := false // Does extended wildcard '*' and '?' match.
+	return deepMatchRune(rname, rpattern, simple)
+}
+
+func deepMatchRune(str, pattern []rune, simple bool) bool {
 	for len(pattern) > 0 {
-		var star bool
-		var chunk string
-		star, chunk, pattern = scanChunk(pattern)
-		if star && chunk == "" {
-			// Trailing * matches rest of string.
-			return true
-		}
-		// Look for match at current position.
-		t, ok := matchChunk(chunk, name)
-		// if we're the last chunk, make sure we've exhausted the name
-		// otherwise we'll give a false result even if we could still match
-		// using the star
-		if ok && (len(t) == 0 || len(pattern) > 0) {
-			name = t
-			continue
-		}
-		if star {
-			// Look for match skipping i+1 bytes.
-			for i := 0; i < len(name); i++ {
-				t, ok := matchChunk(chunk, name[i+1:])
-				if ok {
-					// if we're the last chunk, make sure we exhausted the name
-					if len(pattern) == 0 && len(t) > 0 {
-						continue
-					}
-					name = t
-					continue Pattern
-				}
-			}
-		}
-		return false
-	}
-	return len(name) == 0
-}
-
-// scanChunk gets the next segment of pattern, which is a non-star string
-// possibly preceded by a star.
-func scanChunk(pattern string) (star bool, chunk, rest string) {
-	for len(pattern) > 0 && pattern[0] == '*' {
-		pattern = pattern[1:]
-		star = true
-	}
-	inrange := false
-	var i int
-Scan:
-	for i = 0; i < len(pattern); i++ {
-		switch pattern[i] {
-		case '*':
-			if !inrange {
-				break Scan
-			}
-		}
-	}
-	return star, pattern[0:i], pattern[i:]
-}
-
-// matchChunk checks whether chunk matches the beginning of s.
-// If so, it returns the remainder of s (after the match).
-// Chunk is all single-character operators: literals, char classes, and ?.
-func matchChunk(chunk, s string) (rest string, ok bool) {
-	for len(chunk) > 0 {
-		if len(s) == 0 {
-			return
-		}
-		switch chunk[0] {
-		case '?':
-			_, n := utf8.DecodeRuneInString(s)
-			s = s[n:]
-			chunk = chunk[1:]
+		switch pattern[0] {
 		default:
-			if chunk[0] != s[0] {
-				return
+			if len(str) == 0 || str[0] != pattern[0] {
+				return false
 			}
-			s = s[1:]
-			chunk = chunk[1:]
+		case '?':
+			if len(str) == 0 && !simple {
+				return false
+			}
+		case '*':
+			return deepMatchRune(str, pattern[1:], simple) ||
+				(len(str) > 0 && deepMatchRune(str[1:], pattern, simple))
 		}
+		str = str[1:]
+		pattern = pattern[1:]
 	}
-	return s, true
+	return len(str) == 0 && len(pattern) == 0
 }

pkg/wildcard/match_test.go

@@ -14,16 +14,18 @@
  * limitations under the License.
  */
 
-package wildcard
+package wildcard_test
 
 import (
 	"testing"
+
+	"github.com/minio/minio/pkg/wildcard"
 )
 
-// TestMatchExtended - Tests validate the logic of wild card matching.
-// `MatchExtended` supports '*' and '?' wildcards.
+// TestMatch - Tests validate the logic of wild card matching.
+// `Match` supports '*' and '?' wildcards.
 // Sample usage: In resource matching for bucket policy validation.
-func TestMatchExtended(t *testing.T) {
+func TestMatch(t *testing.T) {
 	testCases := []struct {
 		pattern string
 		text    string
@@ -131,7 +133,8 @@ func TestMatchExtended(t *testing.T) {
 		},
 		// Test case - 15.
 		// Test case where the keyname part of the pattern is expanded in the text.
-		{pattern: "my-bucket/In*/Ka*/Ban",
+		{
+			pattern: "my-bucket/In*/Ka*/Ban",
 			text:    "my-bucket/India/Karnataka/Bangalore",
 			matched: false,
 		},
@@ -144,7 +147,8 @@ func TestMatchExtended(t *testing.T) {
 		},
 		// Test case - 17.
 		// Test case with keyname part being a wildcard in the pattern.
-		{pattern: "my-bucket/*",
+		{
+			pattern: "my-bucket/*",
 			text:    "my-bucket/India",
 			matched: true,
 		},
@@ -367,16 +371,16 @@ func TestMatchExtended(t *testing.T) {
 	}
 	// Iterating over the test cases, call the function under test and asert the output.
 	for i, testCase := range testCases {
-		actualResult := MatchExtended(testCase.pattern, testCase.text)
+		actualResult := wildcard.Match(testCase.pattern, testCase.text)
 		if testCase.matched != actualResult {
 			t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult)
 		}
 	}
 }
 
-// TestMatch - Tests validate the logic of wild card matching.
-// `Match` supports matching for only '*' in the pattern string.
-func TestMatch(t *testing.T) {
+// TestMatchSimple - Tests validate the logic of wild card matching.
+// `MatchSimple` supports matching for only '*' in the pattern string.
+func TestMatchSimple(t *testing.T) {
 	testCases := []struct {
 		pattern string
 		text    string
@@ -484,7 +488,8 @@ func TestMatch(t *testing.T) {
 		},
 		// Test case - 15.
 		// Test case where the keyname part of the pattern is expanded in the text.
-		{pattern: "my-bucket/In*/Ka*/Ban",
+		{
+			pattern: "my-bucket/In*/Ka*/Ban",
 			text:    "my-bucket/India/Karnataka/Bangalore",
 			matched: false,
 		},
@@ -497,7 +502,8 @@ func TestMatch(t *testing.T) {
 		},
 		// Test case - 17.
 		// Test case with keyname part being a wildcard in the pattern.
-		{pattern: "my-bucket/*",
+		{
+			pattern: "my-bucket/*",
 			text:    "my-bucket/India",
 			matched: true,
 		},
@@ -507,10 +513,28 @@ func TestMatch(t *testing.T) {
 			text:    "my-bucket/odo",
 			matched: false,
 		},
+		// Test case - 11.
+		{
+			pattern: "my-bucket/oo?*",
+			text:    "my-bucket/oo???",
+			matched: true,
+		},
+		// Test case - 12:
+		{
+			pattern: "my-bucket/oo??*",
+			text:    "my-bucket/odo",
+			matched: false,
+		},
+		// Test case - 13:
+		{
+			pattern: "?h?*",
+			text:    "?h?hello",
+			matched: true,
+		},
 	}
 	// Iterating over the test cases, call the function under test and asert the output.
 	for i, testCase := range testCases {
-		actualResult := Match(testCase.pattern, testCase.text)
+		actualResult := wildcard.MatchSimple(testCase.pattern, testCase.text)
 		if testCase.matched != actualResult {
 			t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.matched, actualResult)
 		}

vendor/github.com/minio/minio-go/pkg/policy/bucket-policy-condition.go

@@ -0,0 +1,115 @@
+/*
+ * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package policy
+
+import "github.com/minio/minio-go/pkg/set"
+
+// ConditionKeyMap - map of policy condition key and value.
+type ConditionKeyMap map[string]set.StringSet
+
+// Add - adds key and value.  The value is appended If key already exists.
+func (ckm ConditionKeyMap) Add(key string, value set.StringSet) {
+	if v, ok := ckm[key]; ok {
+		ckm[key] = v.Union(value)
+	} else {
+		ckm[key] = set.CopyStringSet(value)
+	}
+}
+
+// Remove - removes value of given key.  If key has empty after removal, the key is also removed.
+func (ckm ConditionKeyMap) Remove(key string, value set.StringSet) {
+	if v, ok := ckm[key]; ok {
+		if value != nil {
+			ckm[key] = v.Difference(value)
+		}
+
+		if ckm[key].IsEmpty() {
+			delete(ckm, key)
+		}
+	}
+}
+
+// RemoveKey - removes key and its value.
+func (ckm ConditionKeyMap) RemoveKey(key string) {
+	if _, ok := ckm[key]; ok {
+		delete(ckm, key)
+	}
+}
+
+// CopyConditionKeyMap - returns new copy of given ConditionKeyMap.
+func CopyConditionKeyMap(condKeyMap ConditionKeyMap) ConditionKeyMap {
+	out := make(ConditionKeyMap)
+
+	for k, v := range condKeyMap {
+		out[k] = set.CopyStringSet(v)
+	}
+
+	return out
+}
+
+// mergeConditionKeyMap - returns a new ConditionKeyMap which contains merged key/value of given two ConditionKeyMap.
+func mergeConditionKeyMap(condKeyMap1 ConditionKeyMap, condKeyMap2 ConditionKeyMap) ConditionKeyMap {
+	out := CopyConditionKeyMap(condKeyMap1)
+
+	for k, v := range condKeyMap2 {
+		if ev, ok := out[k]; ok {
+			out[k] = ev.Union(v)
+		} else {
+			out[k] = set.CopyStringSet(v)
+		}
+	}
+
+	return out
+}
+
+// ConditionMap - map of condition and conditional values.
+type ConditionMap map[string]ConditionKeyMap
+
+// Add - adds condition key and condition value.  The value is appended if key already exists.
+func (cond ConditionMap) Add(condKey string, condKeyMap ConditionKeyMap) {
+	if v, ok := cond[condKey]; ok {
+		cond[condKey] = mergeConditionKeyMap(v, condKeyMap)
+	} else {
+		cond[condKey] = CopyConditionKeyMap(condKeyMap)
+	}
+}
+
+// Remove - removes condition key and its value.
+func (cond ConditionMap) Remove(condKey string) {
+	if _, ok := cond[condKey]; ok {
+		delete(cond, condKey)
+	}
+}
+
+// mergeConditionMap - returns new ConditionMap which contains merged key/value of two ConditionMap.
+func mergeConditionMap(condMap1 ConditionMap, condMap2 ConditionMap) ConditionMap {
+	out := make(ConditionMap)
+
+	for k, v := range condMap1 {
+		out[k] = CopyConditionKeyMap(v)
+	}
+
+	for k, v := range condMap2 {
+		if ev, ok := out[k]; ok {
+			out[k] = mergeConditionKeyMap(ev, v)
+		} else {
+			out[k] = CopyConditionKeyMap(v)
+		}
+	}
+
+	return out
+}

vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go

@@ -0,0 +1,608 @@
+/*
+ * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package policy
+
+import (
+	"reflect"
+	"strings"
+
+	"github.com/minio/minio-go/pkg/set"
+)
+
+// BucketPolicy - Bucket level policy.
+type BucketPolicy string
+
+// Different types of Policies currently supported for buckets.
+const (
+	BucketPolicyNone      BucketPolicy = "none"
+	BucketPolicyReadOnly               = "readonly"
+	BucketPolicyReadWrite              = "readwrite"
+	BucketPolicyWriteOnly              = "writeonly"
+)
+
+// isValidBucketPolicy - Is provided policy value supported.
+func (p BucketPolicy) IsValidBucketPolicy() bool {
+	switch p {
+	case BucketPolicyNone, BucketPolicyReadOnly, BucketPolicyReadWrite, BucketPolicyWriteOnly:
+		return true
+	}
+	return false
+}
+
+// Resource prefix for all aws resources.
+const awsResourcePrefix = "arn:aws:s3:::"
+
+// Common bucket actions for both read and write policies.
+var commonBucketActions = set.CreateStringSet("s3:GetBucketLocation")
+
+// Read only bucket actions.
+var readOnlyBucketActions = set.CreateStringSet("s3:ListBucket")
+
+// Write only bucket actions.
+var writeOnlyBucketActions = set.CreateStringSet("s3:ListBucketMultipartUploads")
+
+// Read only object actions.
+var readOnlyObjectActions = set.CreateStringSet("s3:GetObject")
+
+// Write only object actions.
+var writeOnlyObjectActions = set.CreateStringSet("s3:AbortMultipartUpload", "s3:DeleteObject", "s3:ListMultipartUploadParts", "s3:PutObject")
+
+// Read and write object actions.
+var readWriteObjectActions = readOnlyObjectActions.Union(writeOnlyObjectActions)
+
+// All valid bucket and object actions.
+var validActions = commonBucketActions.
+	Union(readOnlyBucketActions).
+	Union(writeOnlyBucketActions).
+	Union(readOnlyObjectActions).
+	Union(writeOnlyObjectActions)
+
+var startsWithFunc = func(resource string, resourcePrefix string) bool {
+	return strings.HasPrefix(resource, resourcePrefix)
+}
+
+// User - canonical users list.
+type User struct {
+	AWS           set.StringSet `json:"AWS,omitempty"`
+	CanonicalUser set.StringSet `json:"CanonicalUser,omitempty"`
+}
+
+// Statement - minio policy statement
+type Statement struct {
+	Actions    set.StringSet `json:"Action"`
+	Conditions ConditionMap  `json:"Condition,omitempty"`
+	Effect     string
+	Principal  User          `json:"Principal"`
+	Resources  set.StringSet `json:"Resource"`
+	Sid        string
+}
+
+// BucketAccessPolicy - minio policy collection
+type BucketAccessPolicy struct {
+	Version    string      // date in YYYY-MM-DD format
+	Statements []Statement `json:"Statement"`
+}
+
+// isValidStatement - returns whether given statement is valid to process for given bucket name.
+func isValidStatement(statement Statement, bucketName string) bool {
+	if statement.Actions.Intersection(validActions).IsEmpty() {
+		return false
+	}
+
+	if statement.Effect != "Allow" {
+		return false
+	}
+
+	if statement.Principal.AWS == nil || !statement.Principal.AWS.Contains("*") {
+		return false
+	}
+
+	bucketResource := awsResourcePrefix + bucketName
+	if statement.Resources.Contains(bucketResource) {
+		return true
+	}
+
+	if statement.Resources.FuncMatch(startsWithFunc, bucketResource+"/").IsEmpty() {
+		return false
+	}
+
+	return true
+}
+
+// Returns new statements with bucket actions for given policy.
+func newBucketStatement(policy BucketPolicy, bucketName string, prefix string) (statements []Statement) {
+	statements = []Statement{}
+	if policy == BucketPolicyNone || bucketName == "" {
+		return statements
+	}
+
+	bucketResource := set.CreateStringSet(awsResourcePrefix + bucketName)
+
+	statement := Statement{
+		Actions:   commonBucketActions,
+		Effect:    "Allow",
+		Principal: User{AWS: set.CreateStringSet("*")},
+		Resources: bucketResource,
+		Sid:       "",
+	}
+	statements = append(statements, statement)
+
+	if policy == BucketPolicyReadOnly || policy == BucketPolicyReadWrite {
+		statement = Statement{
+			Actions:   readOnlyBucketActions,
+			Effect:    "Allow",
+			Principal: User{AWS: set.CreateStringSet("*")},
+			Resources: bucketResource,
+			Sid:       "",
+		}
+		if prefix != "" {
+			condKeyMap := make(ConditionKeyMap)
+			condKeyMap.Add("s3:prefix", set.CreateStringSet(prefix))
+			condMap := make(ConditionMap)
+			condMap.Add("StringEquals", condKeyMap)
+			statement.Conditions = condMap
+		}
+		statements = append(statements, statement)
+	}
+
+	if policy == BucketPolicyWriteOnly || policy == BucketPolicyReadWrite {
+		statement = Statement{
+			Actions:   writeOnlyBucketActions,
+			Effect:    "Allow",
+			Principal: User{AWS: set.CreateStringSet("*")},
+			Resources: bucketResource,
+			Sid:       "",
+		}
+		statements = append(statements, statement)
+	}
+
+	return statements
+}
+
+// Returns new statements contains object actions for given policy.
+func newObjectStatement(policy BucketPolicy, bucketName string, prefix string) (statements []Statement) {
+	statements = []Statement{}
+	if policy == BucketPolicyNone || bucketName == "" {
+		return statements
+	}
+
+	statement := Statement{
+		Effect:    "Allow",
+		Principal: User{AWS: set.CreateStringSet("*")},
+		Resources: set.CreateStringSet(awsResourcePrefix + bucketName + "/" + prefix + "*"),
+		Sid:       "",
+	}
+
+	if policy == BucketPolicyReadOnly {
+		statement.Actions = readOnlyObjectActions
+	} else if policy == BucketPolicyWriteOnly {
+		statement.Actions = writeOnlyObjectActions
+	} else if policy == BucketPolicyReadWrite {
+		statement.Actions = readWriteObjectActions
+	}
+
+	statements = append(statements, statement)
+	return statements
+}
+
+// Returns new statements for given policy, bucket and prefix.
+func newStatements(policy BucketPolicy, bucketName string, prefix string) (statements []Statement) {
+	statements = []Statement{}
+	ns := newBucketStatement(policy, bucketName, prefix)
+	statements = append(statements, ns...)
+
+	ns = newObjectStatement(policy, bucketName, prefix)
+	statements = append(statements, ns...)
+
+	return statements
+}
+
+// Returns whether given bucket statements are used by other than given prefix statements.
+func getInUsePolicy(statements []Statement, bucketName string, prefix string) (readOnlyInUse, writeOnlyInUse bool) {
+	resourcePrefix := awsResourcePrefix + bucketName + "/"
+	objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*"
+
+	for _, s := range statements {
+		if !s.Resources.Contains(objectResource) && !s.Resources.FuncMatch(startsWithFunc, resourcePrefix).IsEmpty() {
+			if s.Actions.Intersection(readOnlyObjectActions).Equals(readOnlyObjectActions) {
+				readOnlyInUse = true
+			}
+
+			if s.Actions.Intersection(writeOnlyObjectActions).Equals(writeOnlyObjectActions) {
+				writeOnlyInUse = true
+			}
+		}
+		if readOnlyInUse && writeOnlyInUse {
+			break
+		}
+	}
+
+	return readOnlyInUse, writeOnlyInUse
+}
+
+// Removes object actions in given statement.
+func removeObjectActions(statement Statement, objectResource string) Statement {
+	if statement.Conditions == nil {
+		if len(statement.Resources) > 1 {
+			statement.Resources.Remove(objectResource)
+		} else {
+			statement.Actions = statement.Actions.Difference(readOnlyObjectActions)
+			statement.Actions = statement.Actions.Difference(writeOnlyObjectActions)
+		}
+	}
+
+	return statement
+}
+
+// Removes bucket actions for given policy in given statement.
+func removeBucketActions(statement Statement, prefix string, bucketResource string, readOnlyInUse, writeOnlyInUse bool) Statement {
+	removeReadOnly := func() {
+		if !statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) {
+			return
+		}
+
+		if statement.Conditions == nil {
+			statement.Actions = statement.Actions.Difference(readOnlyBucketActions)
+			return
+		}
+
+		if prefix != "" {
+			stringEqualsValue := statement.Conditions["StringEquals"]
+			values := set.NewStringSet()
+			if stringEqualsValue != nil {
+				values = stringEqualsValue["s3:prefix"]
+				if values == nil {
+					values = set.NewStringSet()
+				}
+			}
+
+			values.Remove(prefix)
+
+			if stringEqualsValue != nil {
+				if values.IsEmpty() {
+					delete(stringEqualsValue, "s3:prefix")
+				}
+				if len(stringEqualsValue) == 0 {
+					delete(statement.Conditions, "StringEquals")
+				}
+			}
+
+			if len(statement.Conditions) == 0 {
+				statement.Conditions = nil
+				statement.Actions = statement.Actions.Difference(readOnlyBucketActions)
+			}
+		}
+	}
+
+	removeWriteOnly := func() {
+		if statement.Conditions == nil {
+			statement.Actions = statement.Actions.Difference(writeOnlyBucketActions)
+		}
+	}
+
+	if len(statement.Resources) > 1 {
+		statement.Resources.Remove(bucketResource)
+	} else {
+		if !readOnlyInUse {
+			removeReadOnly()
+		}
+
+		if !writeOnlyInUse {
+			removeWriteOnly()
+		}
+	}
+
+	return statement
+}
+
+// Returns statements containing removed actions/statements for given
+// policy, bucket name and prefix.
+func removeStatements(statements []Statement, bucketName string, prefix string) []Statement {
+	bucketResource := awsResourcePrefix + bucketName
+	objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*"
+	readOnlyInUse, writeOnlyInUse := getInUsePolicy(statements, bucketName, prefix)
+
+	out := []Statement{}
+	readOnlyBucketStatements := []Statement{}
+	s3PrefixValues := set.NewStringSet()
+
+	for _, statement := range statements {
+		if !isValidStatement(statement, bucketName) {
+			out = append(out, statement)
+			continue
+		}
+
+		if statement.Resources.Contains(bucketResource) {
+			if statement.Conditions != nil {
+				statement = removeBucketActions(statement, prefix, bucketResource, false, false)
+			} else {
+				statement = removeBucketActions(statement, prefix, bucketResource, readOnlyInUse, writeOnlyInUse)
+			}
+		} else if statement.Resources.Contains(objectResource) {
+			statement = removeObjectActions(statement, objectResource)
+		}
+
+		if !statement.Actions.IsEmpty() {
+			if statement.Resources.Contains(bucketResource) &&
+				statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) &&
+				statement.Effect == "Allow" &&
+				statement.Principal.AWS.Contains("*") {
+
+				if statement.Conditions != nil {
+					stringEqualsValue := statement.Conditions["StringEquals"]
+					values := set.NewStringSet()
+					if stringEqualsValue != nil {
+						values = stringEqualsValue["s3:prefix"]
+						if values == nil {
+							values = set.NewStringSet()
+						}
+					}
+					s3PrefixValues = s3PrefixValues.Union(values.ApplyFunc(func(v string) string {
+						return bucketResource + "/" + v + "*"
+					}))
+				} else if !s3PrefixValues.IsEmpty() {
+					readOnlyBucketStatements = append(readOnlyBucketStatements, statement)
+					continue
+				}
+			}
+			out = append(out, statement)
+		}
+	}
+
+	skipBucketStatement := true
+	resourcePrefix := awsResourcePrefix + bucketName + "/"
+	for _, statement := range out {
+		if !statement.Resources.FuncMatch(startsWithFunc, resourcePrefix).IsEmpty() &&
+			s3PrefixValues.Intersection(statement.Resources).IsEmpty() {
+			skipBucketStatement = false
+			break
+		}
+	}
+
+	for _, statement := range readOnlyBucketStatements {
+		if skipBucketStatement &&
+			statement.Resources.Contains(bucketResource) &&
+			statement.Effect == "Allow" &&
+			statement.Principal.AWS.Contains("*") &&
+			statement.Conditions == nil {
+			continue
+		}
+
+		out = append(out, statement)
+	}
+
+	if len(out) == 1 {
+		statement := out[0]
+		if statement.Resources.Contains(bucketResource) &&
+			statement.Actions.Intersection(commonBucketActions).Equals(commonBucketActions) &&
+			statement.Effect == "Allow" &&
+			statement.Principal.AWS.Contains("*") &&
+			statement.Conditions == nil {
+			out = []Statement{}
+		}
+	}
+
+	return out
+}
+
+//  Appends given statement into statement list to have unique statements.
+//  - If statement already exists in statement list, it ignores.
+//  - If statement exists with different conditions, they are merged.
+//  - Else the statement is appended to statement list.
+func appendStatement(statements []Statement, statement Statement) []Statement {
+	for i, s := range statements {
+		if s.Actions.Equals(statement.Actions) &&
+			s.Effect == statement.Effect &&
+			s.Principal.AWS.Equals(statement.Principal.AWS) &&
+			reflect.DeepEqual(s.Conditions, statement.Conditions) {
+			statements[i].Resources = s.Resources.Union(statement.Resources)
+			return statements
+		} else if s.Resources.Equals(statement.Resources) &&
+			s.Effect == statement.Effect &&
+			s.Principal.AWS.Equals(statement.Principal.AWS) &&
+			reflect.DeepEqual(s.Conditions, statement.Conditions) {
+			statements[i].Actions = s.Actions.Union(statement.Actions)
+			return statements
+		}
+
+		if s.Resources.Intersection(statement.Resources).Equals(statement.Resources) &&
+			s.Actions.Intersection(statement.Actions).Equals(statement.Actions) &&
+			s.Effect == statement.Effect &&
+			s.Principal.AWS.Intersection(statement.Principal.AWS).Equals(statement.Principal.AWS) {
+			if reflect.DeepEqual(s.Conditions, statement.Conditions) {
+				return statements
+			}
+			if s.Conditions != nil && statement.Conditions != nil {
+				if s.Resources.Equals(statement.Resources) {
+					statements[i].Conditions = mergeConditionMap(s.Conditions, statement.Conditions)
+					return statements
+				}
+			}
+		}
+	}
+
+	if !(statement.Actions.IsEmpty() && statement.Resources.IsEmpty()) {
+		return append(statements, statement)
+	}
+
+	return statements
+}
+
+// Appends two statement lists.
+func appendStatements(statements []Statement, appendStatements []Statement) []Statement {
+	for _, s := range appendStatements {
+		statements = appendStatement(statements, s)
+	}
+
+	return statements
+}
+
+// Returns policy of given bucket statement.
+func getBucketPolicy(statement Statement, prefix string) (commonFound, readOnly, writeOnly bool) {
+	if !(statement.Effect == "Allow" && statement.Principal.AWS.Contains("*")) {
+		return commonFound, readOnly, writeOnly
+	}
+
+	if statement.Actions.Intersection(commonBucketActions).Equals(commonBucketActions) &&
+		statement.Conditions == nil {
+		commonFound = true
+	}
+
+	if statement.Actions.Intersection(writeOnlyBucketActions).Equals(writeOnlyBucketActions) &&
+		statement.Conditions == nil {
+		writeOnly = true
+	}
+
+	if statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) {
+		if prefix != "" && statement.Conditions != nil {
+			if stringEqualsValue, ok := statement.Conditions["StringEquals"]; ok {
+				if s3PrefixValues, ok := stringEqualsValue["s3:prefix"]; ok {
+					if s3PrefixValues.Contains(prefix) {
+						readOnly = true
+					}
+				}
+			} else if stringNotEqualsValue, ok := statement.Conditions["StringNotEquals"]; ok {
+				if s3PrefixValues, ok := stringNotEqualsValue["s3:prefix"]; ok {
+					if !s3PrefixValues.Contains(prefix) {
+						readOnly = true
+					}
+				}
+			}
+		} else if prefix == "" && statement.Conditions == nil {
+			readOnly = true
+		} else if prefix != "" && statement.Conditions == nil {
+			readOnly = true
+		}
+	}
+
+	return commonFound, readOnly, writeOnly
+}
+
+// Returns policy of given object statement.
+func getObjectPolicy(statement Statement) (readOnly bool, writeOnly bool) {
+	if statement.Effect == "Allow" &&
+		statement.Principal.AWS.Contains("*") &&
+		statement.Conditions == nil {
+		if statement.Actions.Intersection(readOnlyObjectActions).Equals(readOnlyObjectActions) {
+			readOnly = true
+		}
+		if statement.Actions.Intersection(writeOnlyObjectActions).Equals(writeOnlyObjectActions) {
+			writeOnly = true
+		}
+	}
+
+	return readOnly, writeOnly
+}
+
+// Returns policy of given bucket name, prefix in given statements.
+func GetPolicy(statements []Statement, bucketName string, prefix string) BucketPolicy {
+	bucketResource := awsResourcePrefix + bucketName
+	objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*"
+
+	bucketCommonFound := false
+	bucketReadOnly := false
+	bucketWriteOnly := false
+	matchedResource := ""
+	objReadOnly := false
+	objWriteOnly := false
+
+	for _, s := range statements {
+		matchedObjResources := set.NewStringSet()
+		if s.Resources.Contains(objectResource) {
+			matchedObjResources.Add(objectResource)
+		} else {
+			matchedObjResources = s.Resources.FuncMatch(resourceMatch, objectResource)
+		}
+
+		if !matchedObjResources.IsEmpty() {
+			readOnly, writeOnly := getObjectPolicy(s)
+			for resource := range matchedObjResources {
+				if len(matchedResource) < len(resource) {
+					objReadOnly = readOnly
+					objWriteOnly = writeOnly
+					matchedResource = resource
+				} else if len(matchedResource) == len(resource) {
+					objReadOnly = objReadOnly || readOnly
+					objWriteOnly = objWriteOnly || writeOnly
+					matchedResource = resource
+				}
+			}
+		} else if s.Resources.Contains(bucketResource) {
+			commonFound, readOnly, writeOnly := getBucketPolicy(s, prefix)
+			bucketCommonFound = bucketCommonFound || commonFound
+			bucketReadOnly = bucketReadOnly || readOnly
+			bucketWriteOnly = bucketWriteOnly || writeOnly
+		}
+	}
+
+	policy := BucketPolicyNone
+	if bucketCommonFound {
+		if bucketReadOnly && bucketWriteOnly && objReadOnly && objWriteOnly {
+			policy = BucketPolicyReadWrite
+		} else if bucketReadOnly && objReadOnly {
+			policy = BucketPolicyReadOnly
+		} else if bucketWriteOnly && objWriteOnly {
+			policy = BucketPolicyWriteOnly
+		}
+	}
+
+	return policy
+}
+
+// Returns new statements containing policy of given bucket name and
+// prefix are appended.
+func SetPolicy(statements []Statement, policy BucketPolicy, bucketName string, prefix string) []Statement {
+	out := removeStatements(statements, bucketName, prefix)
+	// fmt.Println("out = ")
+	// printstatement(out)
+	ns := newStatements(policy, bucketName, prefix)
+	// fmt.Println("ns = ")
+	// printstatement(ns)
+
+	rv := appendStatements(out, ns)
+	// fmt.Println("rv = ")
+	// printstatement(rv)
+
+	return rv
+}
+
+// Match function matches wild cards in 'pattern' for resource.
+func resourceMatch(pattern, resource string) bool {
+	if pattern == "" {
+		return resource == pattern
+	}
+	if pattern == "*" {
+		return true
+	}
+	parts := strings.Split(pattern, "*")
+	if len(parts) == 1 {
+		return resource == pattern
+	}
+	tGlob := strings.HasSuffix(pattern, "*")
+	end := len(parts) - 1
+	if !strings.HasPrefix(resource, parts[0]) {
+		return false
+	}
+	for i := 1; i < end; i++ {
+		if !strings.Contains(resource, parts[i]) {
+			return false
+		}
+		idx := strings.Index(resource, parts[i]) + len(parts[i])
+		resource = resource[idx:]
+	}
+	return tGlob || strings.HasSuffix(resource, parts[end])
+}

vendor/vendor.json

@@ -107,6 +107,12 @@
 			"revision": "db6b4f13442b26995f04b3b2b31b006cae7786e6",
 			"revisionTime": "2016-02-29T08:42:30-08:00"
 		},
+		{
+			"checksumSHA1": "0OZaeJPgMlA2Txn+1yeAIwEpJvM=",
+			"path": "github.com/minio/minio-go/pkg/policy",
+			"revision": "a2e27c84cd20b86cd781b76639781d6366235d0c",
+			"revisionTime": "2016-08-23T00:31:21Z"
+		},
 		{
 			"checksumSHA1": "A8QOw1aWwc+RtjGozY0XeS5varo=",
 			"path": "github.com/minio/minio-go/pkg/set",