Remove double RLocks for GetBucketInfo (#6209)

The error type `hashMismatchError` is lost when the error is received
from a remote disk.

Fixes #6201

.travis.yml

@@ -22,21 +22,9 @@ matrix:
         - make coverage
         - node --version
         - cd browser && yarn && yarn test && cd ..
-    - os: linux-ppc64le
-      env:
-        - ARCH=ppc64le
-      go: 1.10.1
-      script:
-        - make
-        - diff -au <(gofmt -s -d cmd) <(printf "")
-        - diff -au <(gofmt -s -d pkg) <(printf "")
-        - make test GOFLAGS="-timeout 15m -v"
-        - make coverage
-        - node --version
-        - cd browser && yarn && yarn test && cd ..
 
 before_install:
 - nvm install stable
 
 after_success:
-- bash <(curl -s https://codecov.io/bash)
+- bash <(curl -s https://codecov.io/bash)

browser/app/less/inc/list.less

@@ -100,6 +100,12 @@ div.fesl-row {
         }
     }
 
+    .fesl-item-name {
+        a {
+            cursor: default;
+        }
+    }
+
     /*--------------------------
         Icons
     ----------------------------*/

cmd/acl-handlers.go

@@ -54,7 +54,7 @@ type accessControlPolicy struct {
 // This operation uses the ACL
 // subresource to return the ACL of a specified bucket.
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketACL")
+	ctx := newContext(r, w, "GetBucketACL")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -101,7 +101,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 // This operation uses the ACL
 // subresource to return the ACL of a specified object.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetObjectACL")
+	ctx := newContext(r, w, "GetObjectACL")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/admin-handlers.go

@@ -356,7 +356,7 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques
 // ---------
 // Clear locks held on a given bucket, prefix and duration it was held for.
 func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ClearLocks")
+	ctx := newContext(r, w, "ClearLocks")
 
 	// Get object layer instance.
 	objLayer := newObjectLayerFn()
@@ -464,7 +464,7 @@ func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 // sequence. However, if the force-start flag is provided, the server
 // aborts the running heal sequence and starts a new one.
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "Heal")
+	ctx := newContext(r, w, "Heal")
 
 	// Get object layer instance.
 	objLayer := newObjectLayerFn()
@@ -712,7 +712,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	err = json.Unmarshal(configBytes, &config)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
+		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
 		return
 	}
 
@@ -727,6 +727,11 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		}
 	}
 
+	if err := config.Validate(); err != nil {
+		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
+		return
+	}
+
 	// Write config received from request onto a temporary file on
 	// all nodes.
 	tmpFileName := fmt.Sprintf(minioConfigTmpFormat, mustGetUUID())

cmd/admin-handlers_test.go

@@ -38,22 +38,24 @@ import (
 
 var (
 	configJSON = []byte(`{
-	"version": "13",
+	"version": "27",
 	"credential": {
 		"accessKey": "minio",
 		"secretKey": "minio123"
 	},
-	"region": "us-west-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "fatal"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
+	"region": "",
+	"browser": "on",
+	"worm": "off",
+	"domain": "",
+	"storageclass": {
+		"standard": "",
+		"rrs": ""
+	},
+	"cache": {
+		"drives": [],
+		"expiry": 90,
+		"maxuse": 80,
+		"exclude": []
 	},
 	"notify": {
 		"amqp": {
@@ -63,6 +65,7 @@ var (
 				"exchange": "",
 				"routingKey": "",
 				"exchangeType": "",
+				"deliveryMode": 0,
 				"mandatory": false,
 				"immediate": false,
 				"durable": false,
@@ -71,6 +74,47 @@ var (
 				"autoDeleted": false
 			}
 		},
+		"elasticsearch": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"url": "",
+				"index": ""
+			}
+		},
+		"kafka": {
+			"1": {
+				"enable": false,
+				"brokers": null,
+				"topic": ""
+			}
+		},
+		"mqtt": {
+			"1": {
+				"enable": false,
+				"broker": "",
+				"topic": "",
+				"qos": 0,
+				"clientId": "",
+				"username": "",
+				"password": "",
+				"reconnectInterval": 0,
+				"keepAliveInterval": 0
+			}
+		},
+		"mysql": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"dsnString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
 		"nats": {
 			"1": {
 				"enable": false,
@@ -90,24 +134,10 @@ var (
 				}
 			}
 		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
 		"postgresql": {
 			"1": {
 				"enable": false,
+				"format": "",
 				"connectionString": "",
 				"table": "",
 				"host": "",
@@ -117,11 +147,13 @@ var (
 				"database": ""
 			}
 		},
-		"kafka": {
+		"redis": {
 			"1": {
 				"enable": false,
-				"brokers": null,
-				"topic": ""
+				"format": "",
+				"address": "",
+				"password": "",
+				"key": ""
 			}
 		},
 		"webhook": {
@@ -130,8 +162,20 @@ var (
 				"endpoint": ""
 			}
 		}
-	}
-}`)
+	    },
+	    "logger": {
+		"console": {
+		    "enabled": true
+		},
+		"http": {
+		    "1": {
+			"enabled": false,
+			"endpoint": "http://user:example@localhost:9001/api/endpoint"
+		    }
+		}
+	    }
+
+	}`)
 )
 
 // adminXLTestBed - encapsulates subsystems that need to be setup for

cmd/api-errors.go

@@ -40,8 +40,8 @@ type APIErrorResponse struct {
 	XMLName    xml.Name `xml:"Error" json:"-"`
 	Code       string
 	Message    string
-	Key        string
-	BucketName string
+	Key        string `xml:"Key,omitempty" json:"Key,omitempty"`
+	BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"`
 	Resource   string
 	RequestID  string `xml:"RequestId" json:"RequestId"`
 	HostID     string `xml:"HostId" json:"HostId"`
@@ -169,7 +169,6 @@ const (
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
-	ErrPartsSizeUnequal
 	ErrInvalidRequest
 	// Minio storage class error codes
 	ErrInvalidStorageClass
@@ -785,11 +784,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Your metadata headers are not supported.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrPartsSizeUnequal: {
-		Code:           "XMinioPartsSizeUnequal",
-		Description:    "All parts except the last part should be of the same size.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrObjectTampered: {
 		Code:           "XMinioObjectTampered",
 		Description:    errObjectTampered.Error(),
@@ -970,8 +964,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooLarge
 	case UnsupportedMetadata:
 		apiErr = ErrUnsupportedMetadata
-	case PartsSizeUnequal:
-		apiErr = ErrPartsSizeUnequal
 	case BucketPolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
 	case *event.ErrInvalidEventName:
@@ -1012,12 +1004,12 @@ func getAPIError(code APIErrorCode) APIError {
 
 // getErrorResponse gets in standard error and resource value and
 // provides a encodable populated response values
-func getAPIErrorResponse(err APIError, resource string) APIErrorResponse {
+func getAPIErrorResponse(err APIError, resource, requestid string) APIErrorResponse {
 	return APIErrorResponse{
 		Code:      err.Code,
 		Message:   err.Description,
 		Resource:  resource,
-		RequestID: "3L137",
+		RequestID: requestid,
 		HostID:    "3L137",
 	}
 }

cmd/api-headers.go

@@ -34,8 +34,6 @@ func mustGetRequestID(t time.Time) string {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	// Set unique request ID for each reply.
-	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
 	w.Header().Set("Server", globalServerUserAgent)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.

cmd/api-response-multipart.go

@@ -44,7 +44,7 @@ func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err Par
 
 	apiError := getAPIError(toAPIErrorCode(err))
 	// Generate complete multipart error response.
-	errorResponse := getAPIErrorResponse(apiError, r.URL.Path)
+	errorResponse := getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey))
 	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
 	encodedErrorResponse := encodeResponse(cmpErrResp)
 

cmd/api-response.go

@@ -577,7 +577,7 @@ func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *u
 	}
 	apiError := getAPIError(errorCode)
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
+	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
 	encodedErrorResponse := encodeResponse(errorResponse)
 	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
@@ -592,7 +592,7 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, errorCode APIErrorCode
 func writeErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
 	apiError := getAPIError(errorCode)
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
+	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }

cmd/api-router.go

@@ -108,7 +108,7 @@ func registerAPIRouter(router *mux.Router) {
 		// HeadBucket
 		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
 		// PostPolicy
-		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
+		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
 		// DeleteMultipleObjects
 		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
 		// DeleteBucketPolicy

cmd/bucket-handlers-listobjects.go

@@ -54,7 +54,7 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 // NOTE: It is recommended that this API to be used for application development.
 // Minio continues to support ListObjectsV1 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectsV2")
+	ctx := newContext(r, w, "ListObjectsV2")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -123,7 +123,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 // criteria to return a subset of the objects in a bucket.
 //
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectsV1")
+	ctx := newContext(r, w, "ListObjectsV1")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/bucket-handlers.go

@@ -28,7 +28,6 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
 
 	"github.com/gorilla/mux"
 
@@ -87,7 +86,7 @@ func initFederatorBackend(objLayer ObjectLayer) {
 // -------------------------
 // This operation returns bucket location.
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketLocation")
+	ctx := newContext(r, w, "GetBucketLocation")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -103,12 +102,6 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetRLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-	defer bucketLock.RUnlock()
 	getBucketInfo := objectAPI.GetBucketInfo
 	if api.CacheAPI() != nil {
 		getBucketInfo = api.CacheAPI().GetBucketInfo
@@ -141,7 +134,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 // uploads in the response.
 //
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListMultipartUploads")
+	ctx := newContext(r, w, "ListMultipartUploads")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -188,7 +181,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 // This implementation of the GET operation returns a list of all buckets
 // owned by the authenticated sender of the request.
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListBuckets")
+	ctx := newContext(r, w, "ListBuckets")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -244,7 +237,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 // DeleteMultipleObjectsHandler - deletes multiple objects.
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteMultipleObjects")
+	ctx := newContext(r, w, "DeleteMultipleObjects")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -311,34 +304,24 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var wg = &sync.WaitGroup{} // Allocate a new wait group.
-	var dErrs = make([]error, len(deleteObjects.Objects))
-
-	// Delete all requested objects in parallel.
-	for index, object := range deleteObjects.Objects {
-		wg.Add(1)
-		go func(i int, obj ObjectIdentifier) {
-			defer wg.Done()
-			// If the request is denied access, each item
-			// should be marked as 'AccessDenied'
-			if s3Error == ErrAccessDenied {
-				dErrs[i] = PrefixAccessDenied{
-					Bucket: bucket,
-					Object: obj.ObjectName,
-				}
-				return
-			}
-			deleteObject := objectAPI.DeleteObject
-			if api.CacheAPI() != nil {
-				deleteObject = api.CacheAPI().DeleteObject
-			}
-			dErr := deleteObject(ctx, bucket, obj.ObjectName)
-			if dErr != nil {
-				dErrs[i] = dErr
-			}
-		}(index, object)
+	deleteObject := objectAPI.DeleteObject
+	if api.CacheAPI() != nil {
+		deleteObject = api.CacheAPI().DeleteObject
+	}
+
+	var dErrs = make([]error, len(deleteObjects.Objects))
+	for index, object := range deleteObjects.Objects {
+		// If the request is denied access, each item
+		// should be marked as 'AccessDenied'
+		if s3Error == ErrAccessDenied {
+			dErrs[index] = PrefixAccessDenied{
+				Bucket: bucket,
+				Object: object.ObjectName,
+			}
+			continue
+		}
+		dErrs[index] = deleteObject(ctx, bucket, object.ObjectName)
 	}
-	wg.Wait()
 
 	// Collect deleted objects and errors if any.
 	var deletedObjects []ObjectIdentifier
@@ -398,7 +381,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 // ----------
 // This implementation of the PUT operation creates a new bucket for authenticated request
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucket")
+	ctx := newContext(r, w, "PutBucket")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -474,7 +457,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 // This implementation of the POST operation handles object creation with a specified
 // signature policy in multipart/form-data
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PostPolicyBucket")
+	ctx := newContext(r, w, "PostPolicyBucket")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -694,7 +677,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 // have permission to access it. Otherwise, the operation might
 // return responses such as 404 Not Found and 403 Forbidden.
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "HeadBucket")
+	ctx := newContext(r, w, "HeadBucket")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -724,7 +707,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 // DeleteBucketHandler - Delete bucket
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteBucket")
+	ctx := newContext(r, w, "DeleteBucket")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/bucket-notification-handlers.go

@@ -42,7 +42,7 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 // as per http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 // It returns empty configuration if its not set.
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketNotification")
+	ctx := newContext(r, w, "GetBucketNotification")
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
@@ -94,7 +94,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 // PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucketNotification")
+	ctx := newContext(r, w, "PutBucketNotification")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -154,7 +154,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 // ListenBucketNotificationHandler - This HTTP handler sends events to the connected HTTP client.
 // Client should send prefix/suffix object name to match and events to watch as query parameters.
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListenBucketNotification")
+	ctx := newContext(r, w, "ListenBucketNotification")
 
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
@@ -235,7 +235,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
-	if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
+	if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/bucket-policy-handlers.go

@@ -38,7 +38,7 @@ const (
 // PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per
 // https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucketPolicy")
+	ctx := newContext(r, w, "PutBucketPolicy")
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -99,7 +99,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 // DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration.
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteBucketPolicy")
+	ctx := newContext(r, w, "DeleteBucketPolicy")
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -135,7 +135,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 // GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration.
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketPolicy")
+	ctx := newContext(r, w, "GetBucketPolicy")
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-policy-handlers_test.go

@@ -437,11 +437,13 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 		if recV4.Code != testCase.expectedRespStatus {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			var expectedPolicy *policy.Policy
+			expectedPolicy, err = policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}
-			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			var gotPolicy *policy.Policy
+			gotPolicy, err = policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}

cmd/common-main.go

@@ -81,6 +81,20 @@ func initConfig() {
 	}
 }
 
+// Load logger targets based on user's configuration
+func loadLoggers() {
+	if globalServerConfig.Logger.Console.Enabled {
+		// Enable console logging
+		logger.AddTarget(logger.NewConsole())
+	}
+	for _, l := range globalServerConfig.Logger.HTTP {
+		if l.Enabled {
+			// Enable http logging
+			logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
+		}
+	}
+}
+
 func handleCommonCmdArgs(ctx *cli.Context) {
 
 	var configDir string

cmd/config-current.go

@@ -23,6 +23,7 @@ import (
 	"reflect"
 	"sync"
 
+	"github.com/miekg/dns"
 	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/auth"
@@ -40,9 +41,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "26"
+const serverConfigVersion = "27"
 
-type serverConfig = serverConfigV26
+type serverConfig = serverConfigV27
 
 var (
 	// globalServerConfig server config.
@@ -129,6 +130,84 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 	return s.Cache
 }
 
+func (s *serverConfig) Validate() error {
+	if s.Version != serverConfigVersion {
+		return fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, s.Version)
+	}
+
+	// Validate credential fields only when
+	// they are not set via the environment
+	// Error out if global is env credential is not set and config has invalid credential
+	if !globalIsEnvCreds && !s.Credential.IsValid() {
+		return errors.New("invalid credential in config file")
+	}
+
+	// Region: nothing to validate
+	// Browser, Worm, Cache and StorageClass values are already validated during json unmarshal
+
+	if s.Domain != "" {
+		if _, ok := dns.IsDomainName(s.Domain); !ok {
+			return errors.New("invalid domain name")
+		}
+	}
+
+	for _, v := range s.Notify.AMQP {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("amqp: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.Elasticsearch {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("elasticsearch: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.Kafka {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("kafka: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.MQTT {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("mqtt: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.MySQL {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("mysql: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.NATS {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("nats: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.PostgreSQL {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("postgreSQL: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.Redis {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("redis: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.Webhook {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("webhook: %s", err.Error())
+		}
+	}
+
+	return nil
+}
+
 // Save config file to corresponding backend
 func Save(configFile string, data interface{}) error {
 	return quick.SaveConfig(data, configFile, globalEtcdClient)
@@ -181,6 +260,8 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "MySQL Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
 		return "MQTT Notification configuration differs"
+	case !reflect.DeepEqual(s.Logger, t.Logger):
+		return "Logger configuration differs"
 	case reflect.DeepEqual(s, t):
 		return ""
 	default:
@@ -236,6 +317,13 @@ func newServerConfig() *serverConfig {
 	srvCfg.Cache.Exclude = make([]string, 0)
 	srvCfg.Cache.Expiry = globalCacheExpiry
 	srvCfg.Cache.MaxUse = globalCacheMaxUse
+
+	// Console logging is on by default
+	srvCfg.Logger.Console.Enabled = true
+	// Create an example of HTTP logger
+	srvCfg.Logger.HTTP = make(map[string]loggerHTTP)
+	srvCfg.Logger.HTTP["target1"] = loggerHTTP{Endpoint: "https://username:password@example.com/api"}
+
 	return srvCfg
 }
 
@@ -318,15 +406,8 @@ func getValidConfig() (*serverConfig, error) {
 		return nil, err
 	}
 
-	if srvCfg.Version != serverConfigVersion {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
-	}
-
-	// Validate credential fields only when
-	// they are not set via the environment
-	// Error out if global is env credential is not set and config has invalid credential
-	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
-		return nil, errors.New("invalid credential in config file " + getConfigFile())
+	if err = srvCfg.Validate(); err != nil {
+		return nil, err
 	}
 
 	return srvCfg, nil

cmd/config-current_test.go

@@ -174,55 +174,55 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
 
 		// Test 11 - Test AMQP
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
 
 		// Test 12 - Test NATS
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
 		// Test 13 - Test ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
 
 		// Test 14 - Test Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
 		// Test 15 - Test PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
 		// Test 16 - Test Kafka
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
 
 		// Test 17 - Test Webhook
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
 
 		// Test 18 - Test MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
 		// Test 19 - Test Format for MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
 		// Test 20 - Test valid Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
 		// Test 21 - Test Format for PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
 		// Test 22 - Test valid Format for PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
 		// Test 23 - Test Format for ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
 
 		// Test 24 - Test valid Format for ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
 
 		// Test 25 - Test Format for Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
 
 		// Test 26 - Test valid Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
 
 		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
 	}
 
 	for i, testCase := range testCases {
@@ -319,6 +319,18 @@ func TestConfigDiff(t *testing.T) {
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
 			"MQTT Notification configuration differs",
 		},
+		// 16
+		{
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
+			}},
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
+			}},
+			"Logger configuration differs",
+		},
 	}
 
 	for i, testCase := range testCases {

cmd/config-migrate.go

@@ -188,6 +188,11 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
+	case "26":
+		if err = migrateV26ToV27(); err != nil {
+			return err
+		}
+		fallthrough
 	case serverConfigVersion:
 		// No migration needed. this always points to current version.
 		err = nil
@@ -2317,3 +2322,36 @@ func migrateV25ToV26() error {
 	logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
 	return nil
 }
+
+func migrateV26ToV27() error {
+	configFile := getConfigFile()
+
+	// config V27 is backward compatible with V26, load the old
+	// config file in serverConfigV27 struct and put some examples
+	// in the new `logger` field
+	srvConfig := &serverConfigV27{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+
+	if srvConfig.Version != "26" {
+		return nil
+	}
+
+	srvConfig.Version = "27"
+	// Enable console logging by default to avoid breaking users
+	// current deployments
+	srvConfig.Logger.Console.Enabled = true
+	srvConfig.Logger.HTTP = make(map[string]loggerHTTP)
+	srvConfig.Logger.HTTP["1"] = loggerHTTP{}
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "26", "27")
+	return nil
+}

cmd/config-migrate_test.go

@@ -134,10 +134,25 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV21ToV22(); err != nil {
 		t.Fatal("migrate v21 to v22 should succeed when no config file is found")
 	}
+	if err := migrateV22ToV23(); err != nil {
+		t.Fatal("migrate v22 to v23 should succeed when no config file is found")
+	}
+	if err := migrateV23ToV24(); err != nil {
+		t.Fatal("migrate v23 to v24 should succeed when no config file is found")
+	}
+	if err := migrateV24ToV25(); err != nil {
+		t.Fatal("migrate v24 to v25 should succeed when no config file is found")
+	}
+	if err := migrateV25ToV26(); err != nil {
+		t.Fatal("migrate v25 to v26 should succeed when no config file is found")
+	}
+	if err := migrateV26ToV27(); err != nil {
+		t.Fatal("migrate v26 to v27 should succeed when no config file is found")
+	}
 }
 
-// Test if a config migration from v2 to v23 is successfully done
-func TestServerConfigMigrateV2toV23(t *testing.T) {
+// Test if a config migration from v2 to v27 is successfully done
+func TestServerConfigMigrateV2toV27(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
@@ -272,6 +287,18 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV22ToV23(); err == nil {
 		t.Fatal("migrateConfigV22ToV23() should fail with a corrupted json")
 	}
+	if err := migrateV23ToV24(); err == nil {
+		t.Fatal("migrateConfigV23ToV24() should fail with a corrupted json")
+	}
+	if err := migrateV24ToV25(); err == nil {
+		t.Fatal("migrateConfigV24ToV25() should fail with a corrupted json")
+	}
+	if err := migrateV25ToV26(); err == nil {
+		t.Fatal("migrateConfigV25ToV26() should fail with a corrupted json")
+	}
+	if err := migrateV26ToV27(); err == nil {
+		t.Fatal("migrateConfigV26ToV27() should fail with a corrupted json")
+	}
 }
 
 // Test if all migrate code returns error with corrupted config files

cmd/config-versions.go

@@ -657,9 +657,6 @@ type serverConfigV25 struct {
 
 // serverConfigV26 is just like version '25', stores additionally
 // cache max use value in 'CacheConfig'.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV26 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -681,3 +678,47 @@ type serverConfigV26 struct {
 	// Notification queue configuration.
 	Notify notifier `json:"notify"`
 }
+
+type loggerConsole struct {
+	Enabled bool `json:"enabled"`
+}
+
+type loggerHTTP struct {
+	Enabled  bool   `json:"enabled"`
+	Endpoint string `json:"endpoint"`
+}
+
+type loggerConfig struct {
+	Console loggerConsole         `json:"console"`
+	HTTP    map[string]loggerHTTP `json:"http"`
+}
+
+// serverConfigV27 is just like version '26', stores additionally
+// the logger field
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV27 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+}

cmd/crypto/error.go

@@ -16,8 +16,49 @@ package crypto
 
 import "errors"
 
+// Error is the generic type for any error happening during decrypting
+// an object. It indicates that the object itself or its metadata was
+// modified accidentally or maliciously.
+type Error struct{ msg string }
+
+func (e Error) Error() string { return e.msg }
+
 var (
 	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
 	// is not supported.
 	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+
+	// ErrInvalidCustomerAlgorithm indicates that the specified SSE-C algorithm
+	// is not supported.
+	ErrInvalidCustomerAlgorithm = errors.New("The SSE-C algorithm is not supported")
+
+	// ErrMissingCustomerKey indicates that the HTTP headers contains no SSE-C client key.
+	ErrMissingCustomerKey = errors.New("The SSE-C request is missing the customer key")
+
+	// ErrMissingCustomerKeyMD5 indicates that the HTTP headers contains no SSE-C client key
+	// MD5 checksum.
+	ErrMissingCustomerKeyMD5 = errors.New("The SSE-C request is missing the customer key MD5")
+
+	// ErrInvalidCustomerKey indicates that the SSE-C client key is not valid - e.g. not a
+	// base64-encoded string or not 256 bits long.
+	ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
+
+	// ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
+	// computed MD5 sum. This means that the client provided either the wrong key for
+	// a certain MD5 checksum or the wrong MD5 for a certain key.
+	ErrCustomerKeyMD5Mismatch = errors.New("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
+)
+
+var (
+	errMissingInternalIV            = Error{"The object metadata is missing the internal encryption IV"}
+	errMissingInternalSealAlgorithm = Error{"The object metadata is missing the internal seal algorithm"}
+
+	errInvalidInternalIV            = Error{"The internal encryption IV is malformed"}
+	errInvalidInternalSealAlgorithm = Error{"The internal seal algorithm is invalid and not supported"}
+)
+
+var (
+	// errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able
+	// to produce enough random data. This is fatal error and should cause a panic.
+	errOutOfEntropy = errors.New("Unable to read enough randomness from the system")
 )

cmd/crypto/header.go

@@ -15,12 +15,43 @@
 package crypto
 
 import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
 	"net/http"
 )
 
 // SSEHeader is the general AWS SSE HTTP header key.
 const SSEHeader = "X-Amz-Server-Side-Encryption"
 
+const (
+	// SSECAlgorithm is the HTTP header key referencing
+	// the SSE-C algorithm.
+	SSECAlgorithm = SSEHeader + "-Customer-Algorithm"
+
+	// SSECKey is the HTTP header key referencing the
+	// SSE-C client-provided key..
+	SSECKey = SSEHeader + "-Customer-Key"
+
+	// SSECKeyMD5 is the HTTP header key referencing
+	// the MD5 sum of the client-provided key.
+	SSECKeyMD5 = SSEHeader + "-Customer-Key-Md5"
+)
+
+const (
+	// SSECopyAlgorithm is the HTTP header key referencing
+	// the SSE-C algorithm for SSE-C copy requests.
+	SSECopyAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
+
+	// SSECopyKey is the HTTP header key referencing the SSE-C
+	// client-provided key for SSE-C copy requests.
+	SSECopyKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
+
+	// SSECopyKeyMD5 is the HTTP header key referencing the
+	// MD5 sum of the client key for SSE-C copy requests.
+	SSECopyKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
+)
+
 // SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
 // For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
 // For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
@@ -39,11 +70,107 @@ func (s3) IsRequested(h http.Header) bool {
 	return ok
 }
 
-// Parse parses the SSE-S3 related HTTP headers and checks
+// ParseHTTP parses the SSE-S3 related HTTP headers and checks
 // whether they contain valid values.
-func (s3) Parse(h http.Header) (err error) {
+func (s3) ParseHTTP(h http.Header) (err error) {
 	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
 		err = ErrInvalidEncryptionMethod
 	}
 	return
 }
+
+var (
+	// SSEC represents AWS SSE-C. It provides functionality to handle
+	// SSE-C requests.
+	SSEC = ssec{}
+
+	// SSECopy represents AWS SSE-C for copy requests. It provides
+	// functionality to handle SSE-C copy requests.
+	SSECopy = ssecCopy{}
+)
+
+type ssec struct{}
+type ssecCopy struct{}
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C header. SSE-C copy headers are ignored.
+func (ssec) IsRequested(h http.Header) bool {
+	if _, ok := h[SSECAlgorithm]; ok {
+		return true
+	}
+	if _, ok := h[SSECKey]; ok {
+		return true
+	}
+	if _, ok := h[SSECKeyMD5]; ok {
+		return true
+	}
+	return false
+}
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C copy header. Regular SSE-C headers
+// are ignored.
+func (ssecCopy) IsRequested(h http.Header) bool {
+	if _, ok := h[SSECopyAlgorithm]; ok {
+		return true
+	}
+	if _, ok := h[SSECopyKey]; ok {
+		return true
+	}
+	if _, ok := h[SSECopyKeyMD5]; ok {
+		return true
+	}
+	return false
+}
+
+// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
+// on success. SSE-C copy headers are ignored.
+func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
+	defer h.Del(SSECKey) // remove SSE-C key from headers after parsing
+	if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
+		return key, ErrInvalidCustomerAlgorithm
+	}
+	if h.Get(SSECKey) == "" {
+		return key, ErrMissingCustomerKey
+	}
+	if h.Get(SSECKeyMD5) == "" {
+		return key, ErrMissingCustomerKeyMD5
+	}
+
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECKey))
+	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+		return key, ErrInvalidCustomerKey
+	}
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECKeyMD5))
+	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+		return key, ErrCustomerKeyMD5Mismatch
+	}
+	copy(key[:], clientKey)
+	return key, nil
+}
+
+// ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
+// on success. Regular SSE-C headers are ignored.
+func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
+	defer h.Del(SSECopyKey) // remove SSE-C copy key of source object from headers after parsing
+	if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
+		return key, ErrInvalidCustomerAlgorithm
+	}
+	if h.Get(SSECopyKey) == "" {
+		return key, ErrMissingCustomerKey
+	}
+	if h.Get(SSECopyKeyMD5) == "" {
+		return key, ErrMissingCustomerKeyMD5
+	}
+
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKey))
+	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+		return key, ErrInvalidCustomerKey
+	}
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKeyMD5))
+	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+		return key, ErrCustomerKeyMD5Mismatch
+	}
+	copy(key[:], clientKey)
+	return key, nil
+}

cmd/crypto/header_test.go

@@ -19,7 +19,7 @@ import (
 	"testing"
 )
 
-var isRequestedTests = []struct {
+var s3IsRequestedTests = []struct {
 	Header   http.Header
 	Expected bool
 }{
@@ -30,14 +30,14 @@ var isRequestedTests = []struct {
 }
 
 func TestS3IsRequested(t *testing.T) {
-	for i, test := range isRequestedTests {
+	for i, test := range s3IsRequestedTests {
 		if got := S3.IsRequested(test.Header); got != test.Expected {
 			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
 }
 
-var parseTests = []struct {
+var s3ParseTests = []struct {
 	Header      http.Header
 	ExpectedErr error
 }{
@@ -48,9 +48,257 @@ var parseTests = []struct {
 }
 
 func TestS3Parse(t *testing.T) {
-	for i, test := range parseTests {
-		if err := S3.Parse(test.Header); err != test.ExpectedErr {
+	for i, test := range s3ParseTests {
+		if err := S3.ParseHTTP(test.Header); err != test.ExpectedErr {
 			t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
 		}
 	}
 }
+
+var ssecIsRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{}, Expected: false},                                                                                                     // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true},                                 // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true},                 // 3
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{""},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{""},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{""},
+		},
+		Expected: true,
+	}, // 4
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: true,
+	}, // 5
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: false,
+	}, // 6
+}
+
+func TestSSECIsRequested(t *testing.T) {
+	for i, test := range ssecIsRequestedTests {
+		if got := SSEC.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var ssecCopyIsRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{}, Expected: false},                                                                                                                 // 0
+	{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true},                                 // 1
+	{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
+	{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true},                 // 3
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{""},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{""},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{""},
+		},
+		Expected: true,
+	}, // 4
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: true,
+	}, // 5
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: false,
+	}, // 6
+}
+
+func TestSSECopyIsRequested(t *testing.T) {
+	for i, test := range ssecCopyIsRequestedTests {
+		if got := SSECopy.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var ssecParseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: nil, // 0
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerAlgorithm, // 1
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{""}, // no client key
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrMissingCustomerKey, // 2
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerKey, // 3
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{""}, // no key MD5
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5, // 4
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6
+	},
+}
+
+func TestSSECParse(t *testing.T) {
+	var zeroKey [32]byte
+	for i, test := range ssecParseTests {
+		key, err := SSEC.ParseHTTP(test.Header)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+
+		if err != nil && key != zeroKey {
+			t.Errorf("Test %d: parsing failed and client key is not zero key", i)
+		}
+		if err == nil && key == zeroKey {
+			t.Errorf("Test %d: parsed client key is zero key", i)
+		}
+		if _, ok := test.Header[SSECKey]; ok {
+			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
+		}
+	}
+}
+
+var ssecCopyParseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: nil, // 0
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerAlgorithm, // 1
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{""}, // no client key
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrMissingCustomerKey, // 2
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerKey, // 3
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{""}, // no key MD5
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5, // 4
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6
+	},
+}
+
+func TestSSECopyParse(t *testing.T) {
+	var zeroKey [32]byte
+	for i, test := range ssecCopyParseTests {
+		key, err := SSECopy.ParseHTTP(test.Header)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+
+		if err != nil && key != zeroKey {
+			t.Errorf("Test %d: parsing failed and client key is not zero key", i)
+		}
+		if err == nil && key == zeroKey {
+			t.Errorf("Test %d: parsed client key is zero key", i)
+		}
+		if _, ok := test.Header[SSECKey]; ok {
+			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
+		}
+	}
+}

cmd/crypto/key.go

@@ -21,8 +21,9 @@ import (
 	"crypto/rand"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
-	"path/filepath"
+	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	sha256 "github.com/minio/sha256-simd"
@@ -42,42 +43,78 @@ func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
 	}
 	var nonce [32]byte
 	if _, err := io.ReadFull(random, nonce[:]); err != nil {
-		logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
 	}
 	sha := sha256.New()
 	sha.Write(extKey[:])
 	sha.Write(nonce[:])
 	sha.Sum(key[:0])
-	return
+	return key
+}
+
+// SealedKey represents a sealed object key. It can be stored
+// at an untrusted location.
+type SealedKey struct {
+	Key       [64]byte // The encrypted and authenticted object-key.
+	IV        [32]byte // The random IV used to encrypt the object-key.
+	Algorithm string   // The sealing algorithm used to encrypt the object key.
 }
 
 // Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
-// key is also cryptographically bound to the object's path (bucket/object).
-func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
-	var sealedKey bytes.Buffer
+// key is also cryptographically bound to the object's path (bucket/object) and the
+// domain (SSE-C or SSE-S3).
+func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) SealedKey {
+	var (
+		sealingKey   [32]byte
+		encryptedKey bytes.Buffer
+	)
 	mac := hmac.New(sha256.New, extKey[:])
 	mac.Write(iv[:])
-	mac.Write([]byte(filepath.Join(bucket, object)))
+	mac.Write([]byte(domain))
+	mac.Write([]byte(SealAlgorithm))
+	mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object'
+	mac.Sum(sealingKey[:0])
 
-	if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
+	if n, err := sio.Encrypt(&encryptedKey, bytes.NewReader(key[:]), sio.Config{Key: sealingKey[:]}); n != 64 || err != nil {
 		logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
 	}
-	return sealedKey.Bytes()
+	sealedKey := SealedKey{
+		IV:        iv,
+		Algorithm: SealAlgorithm,
+	}
+	copy(sealedKey.Key[:], encryptedKey.Bytes())
+	return sealedKey
 }
 
-// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
-// is cryptographically bound to the object's path the same bucket/object as during sealing
+// Unseal decrypts a sealed key using the 256 bit external key. Since the sealed key
+// may be cryptographically bound to the object's path the same bucket/object as during sealing
 // must be provided. On success the ObjectKey contains the decrypted sealed key.
-func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
-	var unsealedKey bytes.Buffer
-	mac := hmac.New(sha256.New, extKey[:])
-	mac.Write(iv[:])
-	mac.Write([]byte(filepath.Join(bucket, object)))
+func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error {
+	var (
+		unsealConfig sio.Config
+		decryptedKey bytes.Buffer
+	)
+	switch sealedKey.Algorithm {
+	default:
+		return Error{fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm)}
+	case SealAlgorithm:
+		mac := hmac.New(sha256.New, extKey[:])
+		mac.Write(sealedKey.IV[:])
+		mac.Write([]byte(domain))
+		mac.Write([]byte(SealAlgorithm))
+		mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object'
+		unsealConfig = sio.Config{MinVersion: sio.Version20, Key: mac.Sum(nil)}
+	case InsecureSealAlgorithm:
+		sha := sha256.New()
+		sha.Write(extKey[:])
+		sha.Write(sealedKey.IV[:])
+		unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)}
+	}
 
-	if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
+	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
 		return err // TODO(aead): upgrade sio to use sio.Error
 	}
-	copy(key[:], unsealedKey.Bytes())
+	copy(key[:], decryptedKey.Bytes())
 	return nil
 }
 
@@ -89,5 +126,5 @@ func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
 	mac := hmac.New(sha256.New, key[:])
 	mac.Write(bin[:])
 	mac.Sum(partKey[:0])
-	return
+	return partKey
 }

cmd/crypto/key_test.go

@@ -20,6 +20,8 @@ import (
 	"encoding/hex"
 	"io"
 	"testing"
+
+	"github.com/minio/minio/cmd/logger"
 )
 
 var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
@@ -37,14 +39,18 @@ var generateKeyTests = []struct {
 	Random     io.Reader
 	ShouldPass bool
 }{
-	{ExtKey: [32]byte{}, Random: nil, ShouldPass: true},             // 0
-	{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true},     // 1
-	{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
-	// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
+	{ExtKey: [32]byte{}, Random: nil, ShouldPass: true},              // 0
+	{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true},      // 1
+	{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true},  // 2
+	{ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3
 }
 
 func TestGenerateKey(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+
 	for i, test := range generateKeyTests {
+		i, test := i, test
 		func() {
 			defer recoverTest(i, test.ShouldPass, t)
 			key := GenerateKey(test.ExtKey, test.Random)
@@ -56,37 +62,37 @@ func TestGenerateKey(t *testing.T) {
 }
 
 var sealUnsealKeyTests = []struct {
-	SealExtKey, SealIV     [32]byte
-	SealBucket, SealObject string
+	SealExtKey, SealIV                 [32]byte
+	SealDomain, SealBucket, SealObject string
 
-	UnsealExtKey, UnsealIV     [32]byte
-	UnsealBucket, UnsealObject string
+	UnsealExtKey                             [32]byte
+	UnsealDomain, UnsealBucket, UnsealObject string
 
 	ShouldPass bool
 }{
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object",
 		ShouldPass: true,
 	}, // 0
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{1}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different ext-key
 		ShouldPass: false,
 	}, // 1
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-S3", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different domain
 		ShouldPass: false,
 	}, // 2
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "Bucket", UnsealObject: "object", // different bucket
 		ShouldPass: false,
 	}, // 3
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "Object", // different object
 		ShouldPass: false,
 	}, // 4
 }
@@ -94,13 +100,22 @@ var sealUnsealKeyTests = []struct {
 func TestSealUnsealKey(t *testing.T) {
 	for i, test := range sealUnsealKeyTests {
 		key := GenerateKey(test.SealExtKey, rand.Reader)
-		sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
-		if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
+		sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealDomain, test.SealBucket, test.SealObject)
+		if err := key.Unseal(test.UnsealExtKey, sealedKey, test.UnsealDomain, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
 			t.Errorf("Test %d should fail but passed successfully", i)
 		} else if err != nil && test.ShouldPass {
 			t.Errorf("Test %d should pass put failed: %v", i, err)
 		}
 	}
+
+	// Test legacy InsecureSealAlgorithm
+	var extKey, iv [32]byte
+	key := GenerateKey(extKey, rand.Reader)
+	sealedKey := key.Seal(extKey, iv, "SSE-S3", "bucket", "object")
+	sealedKey.Algorithm = InsecureSealAlgorithm
+	if err := key.Unseal(extKey, sealedKey, "SSE-S3", "bucket", "object"); err == nil {
+		t.Errorf("'%s' test succeeded but it should fail because the legacy algorithm was used", sealedKey.Algorithm)
+	}
 }
 
 var derivePartKeyTest = []struct {

cmd/crypto/kms.go

@@ -0,0 +1,138 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+// Context is a list of key-value pairs cryptographically
+// associated with a certain object.
+type Context map[string]string
+
+// WriteTo writes the context in a canonical from to w.
+// It returns the number of bytes and the first error
+// encounter during writing to w, if any.
+//
+// WriteTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+func (c Context) WriteTo(w io.Writer) (n int64, err error) {
+	sortedKeys := make(sort.StringSlice, 0, len(c))
+	for k := range c {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Sort(sortedKeys)
+
+	nn, err := io.WriteString(w, "{")
+	if err != nil {
+		return n + int64(nn), err
+	}
+	n += int64(nn)
+	for i, k := range sortedKeys {
+		s := fmt.Sprintf("\"%s\":\"%s\",", k, c[k])
+		if i == len(sortedKeys)-1 {
+			s = s[:len(s)-1] // remove last ','
+		}
+
+		nn, err = io.WriteString(w, s)
+		if err != nil {
+			return n + int64(nn), err
+		}
+		n += int64(nn)
+	}
+	nn, err = io.WriteString(w, "}")
+	return n + int64(nn), err
+}
+
+// KMS represents an active and authenticted connection
+// to a Key-Management-Service. It supports generating
+// data key generation and unsealing of KMS-generated
+// data keys.
+type KMS interface {
+	// GenerateKey generates a new random data key using
+	// the master key referenced by the keyID. It returns
+	// the plaintext key and the sealed plaintext key
+	// on success.
+	//
+	// The context is cryptographically bound to the
+	// generated key. The same context must be provided
+	// again to unseal the generated key.
+	GenerateKey(keyID string, context Context) (key [32]byte, sealedKey []byte, err error)
+
+	// UnsealKey unseals the sealedKey using the master key
+	// referenced by the keyID. The provided context must
+	// match the context used to generate the sealed key.
+	UnsealKey(keyID string, sealedKey []byte, context Context) (key [32]byte, err error)
+}
+
+type masterKeyKMS struct {
+	masterKey [32]byte
+}
+
+// NewKMS returns a basic KMS implementation from a single 256 bit master key.
+//
+// The KMS accepts any keyID but binds the keyID and context cryptographically
+// to the generated keys.
+func NewKMS(key [32]byte) KMS { return &masterKeyKMS{masterKey: key} }
+
+func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
+	if _, err = io.ReadFull(rand.Reader, key[:]); err != nil {
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
+	}
+
+	var (
+		buffer     bytes.Buffer
+		derivedKey = kms.deriveKey(keyID, ctx)
+	)
+	if n, err := sio.Encrypt(&buffer, bytes.NewReader(key[:]), sio.Config{Key: derivedKey[:]}); err != nil || n != 64 {
+		logger.CriticalIf(context.Background(), errors.New("KMS: unable to encrypt data key"))
+	}
+	sealedKey = buffer.Bytes()
+	return key, sealedKey, nil
+}
+
+func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
+	var (
+		buffer     bytes.Buffer
+		derivedKey = kms.deriveKey(keyID, ctx)
+	)
+	if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
+		return key, err // TODO(aead): upgrade sio to use sio.Error
+	}
+	copy(key[:], buffer.Bytes())
+	return key, nil
+}
+
+func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte) {
+	if context == nil {
+		context = Context{}
+	}
+	mac := hmac.New(sha256.New, kms.masterKey[:])
+	mac.Write([]byte(keyID))
+	context.WriteTo(mac)
+	mac.Sum(key[:0])
+	return key
+}

cmd/crypto/kms_test.go

@@ -0,0 +1,84 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"path"
+	"strings"
+	"testing"
+)
+
+var masterKeyKMSTests = []struct {
+	GenKeyID, UnsealKeyID     string
+	GenContext, UnsealContext Context
+
+	ShouldFail bool
+}{
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{}, UnsealContext: nil, ShouldFail: false},                                                                                     // 0
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: false},                                                               // 1
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"bucket": "object"}, ShouldFail: false},                           // 2
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: false}, // 3
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "0", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: false},                       // 4
+
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7e", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: true},                                                               // 5
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"Bucket": "object"}, ShouldFail: true},                           // 6
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "Object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: true}, // 7
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "1", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: true},                       // 8
+}
+
+func TestMasterKeyKMS(t *testing.T) {
+	kms := NewKMS([32]byte{})
+	for i, test := range masterKeyKMSTests {
+		key, sealedKey, err := kms.GenerateKey(test.GenKeyID, test.GenContext)
+		if err != nil {
+			t.Errorf("Test %d: KMS failed to generate key: %v", i, err)
+		}
+		unsealedKey, err := kms.UnsealKey(test.UnsealKeyID, sealedKey, test.UnsealContext)
+		if err != nil && !test.ShouldFail {
+			t.Errorf("Test %d: KMS failed to unseal the generated key: %v", i, err)
+		}
+		if err == nil && test.ShouldFail {
+			t.Errorf("Test %d: KMS unsealed the generated successfully but should have failed", i)
+		}
+		if !test.ShouldFail && !bytes.Equal(key[:], unsealedKey[:]) {
+			t.Errorf("Test %d: The generated and unsealed key differ", i)
+		}
+	}
+}
+
+var contextWriteToTests = []struct {
+	Context      Context
+	ExpectedJSON string
+}{
+	{Context: Context{}, ExpectedJSON: "{}"},                                                    // 0
+	{Context: Context{"a": "b"}, ExpectedJSON: `{"a":"b"}`},                                     // 1
+	{Context: Context{"a": "b", "c": "d"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 2
+	{Context: Context{"c": "d", "a": "b"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 3
+	{Context: Context{"0": "1", "-": "2", ".": "#"}, ExpectedJSON: `{"-":"2",".":"#","0":"1"}`}, // 4
+}
+
+func TestContextWriteTo(t *testing.T) {
+	for i, test := range contextWriteToTests {
+		var jsonContext strings.Builder
+		if _, err := test.Context.WriteTo(&jsonContext); err != nil {
+			t.Errorf("Test %d: Failed to encode context: %v", i, err)
+			continue
+		}
+		if s := jsonContext.String(); s != test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
+		}
+	}
+}

cmd/crypto/metadata.go

@@ -0,0 +1,213 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// IsMultiPart returns true if the object metadata indicates
+// that it was uploaded using some form of server-side-encryption
+// and the S3 multipart API.
+func IsMultiPart(metadata map[string]string) bool {
+	if _, ok := metadata[SSEMultipart]; ok {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that it was uploaded using some form of server-side-encryption.
+//
+// IsEncrypted only checks whether the metadata contains at least
+// one entry indicating SSE-C or SSE-S3.
+func IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[SSEIV]; ok {
+		return true
+	}
+	if _, ok := metadata[SSESealAlgorithm]; ok {
+		return true
+	}
+	if IsMultiPart(metadata) {
+		return true
+	}
+	if S3.IsEncrypted(metadata) {
+		return true
+	}
+	if SSEC.IsEncrypted(metadata) {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-S3.
+func (s3) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[S3SealedKey]; ok {
+		return true
+	}
+	if _, ok := metadata[S3KMSKeyID]; ok {
+		return true
+	}
+	if _, ok := metadata[S3KMSSealedKey]; ok {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-C.
+func (ssec) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[SSECSealedKey]; ok {
+		return true
+	}
+	return false
+}
+
+// CreateMultipartMetadata adds the multipart flag entry to metadata
+// and returns modifed metadata. It allocates a new metadata map if
+// metadata is nil.
+func CreateMultipartMetadata(metadata map[string]string) map[string]string {
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+	metadata[SSEMultipart] = ""
+	return metadata
+}
+
+// CreateMetadata encodes the keyID, the sealed kms data key and the sealed key
+// into the metadata and returns the modified metadata. It allocates a new
+// metadata map if metadata is nil.
+func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+	}
+
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+	metadata[S3KMSKeyID] = keyID
+	metadata[SSESealAlgorithm] = sealedKey.Algorithm
+	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[S3SealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-S3 related values from the object metadata
+// and checks whether they are well-formed. It returns the KMS key-ID, the
+// sealed KMS key and the sealed object key on success.
+func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[SSEIV]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[SSESealAlgorithm]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[S3SealedKey]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-S3"}
+	}
+	keyID, ok = metadata[S3KMSKeyID]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"}
+	}
+	b64KMSSealedKey, ok := metadata[S3KMSSealedKey]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"}
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm {
+		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return keyID, kmsKey, sealedKey, Error{"The internal sealed key for SSE-S3 is invalid"}
+	}
+	kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+	if err != nil {
+		return keyID, kmsKey, sealedKey, Error{"The internal sealed KMS data key for SSE-S3 is invalid"}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return keyID, kmsKey, sealedKey, nil
+}
+
+// CreateMetadata encodes the sealed key into the metadata and returns the modified metadata.
+// It allocates a new metadata map if metadata is nil.
+func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
+	}
+
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+	metadata[SSESealAlgorithm] = SealAlgorithm
+	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[SSECSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-C related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success.
+func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[SSEIV]
+	if !ok {
+		return sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[SSESealAlgorithm]
+	if !ok {
+		return sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[SSECSealedKey]
+	if !ok {
+		return sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-C"}
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
+		return sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return sealedKey, Error{"The internal sealed key for SSE-C is invalid"}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return sealedKey, nil
+}

cmd/crypto/metadata_test.go

@@ -0,0 +1,361 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"encoding/base64"
+	"testing"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+var isMultipartTests = []struct {
+	Metadata  map[string]string
+	Multipart bool
+}{
+	{Multipart: true, Metadata: map[string]string{SSEMultipart: ""}},                           // 0
+	{Multipart: true, Metadata: map[string]string{"X-Minio-Internal-Encrypted-Multipart": ""}}, // 1
+	{Multipart: true, Metadata: map[string]string{SSEMultipart: "some-value"}},                 // 2
+	{Multipart: false, Metadata: map[string]string{"": ""}},                                    // 3
+	{Multipart: false, Metadata: map[string]string{"X-Minio-Internal-EncryptedMultipart": ""}}, // 4
+}
+
+func TestIsMultipart(t *testing.T) {
+	for i, test := range isMultipartTests {
+		if isMultipart := IsMultiPart(test.Metadata); isMultipart != test.Multipart {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isMultipart, test.Multipart)
+		}
+	}
+}
+
+var isEncryptedTests = []struct {
+	Metadata  map[string]string
+	Encrypted bool
+}{
+	{Encrypted: true, Metadata: map[string]string{SSEMultipart: ""}},                               // 0
+	{Encrypted: true, Metadata: map[string]string{SSEIV: ""}},                                      // 1
+	{Encrypted: true, Metadata: map[string]string{SSESealAlgorithm: ""}},                           // 2
+	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
+	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
+	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
+	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
+	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
+}
+
+func TestIsEncrypted(t *testing.T) {
+	for i, test := range isEncryptedTests {
+		if isEncrypted := IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
+		}
+	}
+}
+
+var s3IsEncryptedTests = []struct {
+	Metadata  map[string]string
+	Encrypted bool
+}{
+	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
+	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
+	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
+	{Encrypted: false, Metadata: map[string]string{SSECSealedKey: ""}},                             // 3
+	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
+	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
+	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
+	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
+}
+
+func TestS3IsEncrypted(t *testing.T) {
+	for i, test := range s3IsEncryptedTests {
+		if isEncrypted := S3.IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
+		}
+	}
+}
+
+var ssecIsEncryptedTests = []struct {
+	Metadata  map[string]string
+	Encrypted bool
+}{
+	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
+	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
+	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
+	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
+	{Encrypted: false, Metadata: map[string]string{S3SealedKey: ""}},                               // 4
+	{Encrypted: false, Metadata: map[string]string{S3KMSKeyID: ""}},                                // 5
+	{Encrypted: false, Metadata: map[string]string{S3KMSSealedKey: ""}},                            // 6
+	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
+	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
+}
+
+func TestSSECIsEncrypted(t *testing.T) {
+	for i, test := range ssecIsEncryptedTests {
+		if isEncrypted := SSEC.IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
+		}
+	}
+}
+
+var s3ParseMetadataTests = []struct {
+	Metadata    map[string]string
+	ExpectedErr error
+
+	DataKey   []byte
+	KeyID     string
+	SealedKey SealedKey
+}{
+	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}}, // 0
+	{
+		ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 1
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-S3"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 2
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 3
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
+		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 4
+	{
+		ExpectedErr: errInvalidInternalIV,
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: ""},
+		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 5
+	{
+		ExpectedErr: errInvalidInternalSealAlgorithm,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: "",
+		},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 6
+	{
+		ExpectedErr: Error{"The internal sealed key for SSE-S3 is invalid"},
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
+			S3KMSKeyID: "", S3KMSSealedKey: "",
+		},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 7
+	{
+		ExpectedErr: Error{"The internal sealed KMS data key for SSE-S3 is invalid"},
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
+			S3KMSSealedKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
+		},
+		DataKey: []byte{}, KeyID: "key-1", SealedKey: SealedKey{},
+	}, // 8
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "", S3KMSSealedKey: "",
+		},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{Algorithm: SealAlgorithm},
+	}, // 9
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: SealAlgorithm,
+			S3SealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), S3KMSKeyID: "key-1",
+			S3KMSSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
+		},
+		DataKey: make([]byte, 48), KeyID: "key-1", SealedKey: SealedKey{Algorithm: SealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
+	}, // 10
+}
+
+func TestS3ParseMetadata(t *testing.T) {
+	for i, test := range s3ParseMetadataTests {
+		keyID, dataKey, sealedKey, err := S3.ParseMetadata(test.Metadata)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if !bytes.Equal(dataKey, test.DataKey) {
+			t.Errorf("Test %d: got data key '%v' - want data key '%v'", i, dataKey, test.DataKey)
+		}
+		if keyID != test.KeyID {
+			t.Errorf("Test %d: got key-ID '%v' - want key-ID '%v'", i, keyID, test.KeyID)
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: got sealed key '%v' - want sealed key '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: got sealed key IV '%v' - want sealed key IV '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+	}
+}
+
+var ssecParseMetadataTests = []struct {
+	Metadata    map[string]string
+	ExpectedErr error
+
+	SealedKey SealedKey
+}{
+	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
+	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-C"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
+	}, // 2
+	{
+		ExpectedErr: errInvalidInternalIV,
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", SSECSealedKey: ""}, SealedKey: SealedKey{},
+	}, // 3
+	{
+		ExpectedErr: errInvalidInternalSealAlgorithm,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", SSECSealedKey: "",
+		},
+		SealedKey: SealedKey{},
+	}, // 4
+	{
+		ExpectedErr: Error{"The internal sealed key for SSE-C is invalid"},
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
+		},
+		SealedKey: SealedKey{},
+	}, // 5
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			SSECSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+		},
+		SealedKey: SealedKey{Algorithm: SealAlgorithm},
+	}, // 6
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: InsecureSealAlgorithm,
+			SSECSealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
+		},
+		SealedKey: SealedKey{Algorithm: InsecureSealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
+	}, // 7
+}
+
+func TestCreateMultipartMetadata(t *testing.T) {
+	metadata := CreateMultipartMetadata(nil)
+	if v, ok := metadata[SSEMultipart]; !ok || v != "" {
+		t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", SSEMultipart, v, "")
+	}
+}
+
+func TestSSECParseMetadata(t *testing.T) {
+	for i, test := range ssecParseMetadataTests {
+		sealedKey, err := SSEC.ParseMetadata(test.Metadata)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: got sealed key '%v' - want sealed key '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: got sealed key IV '%v' - want sealed key IV '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+	}
+}
+
+var s3CreateMetadataTests = []struct {
+	KeyID         string
+	SealedDataKey []byte
+	SealedKey     SealedKey
+}{
+	{KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}},
+}
+
+func TestS3CreateMetadata(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+	for i, test := range s3CreateMetadataTests {
+		metadata := S3.CreateMetadata(nil, test.KeyID, test.SealedDataKey, test.SealedKey)
+		keyID, kmsKey, sealedKey, err := S3.ParseMetadata(metadata)
+		if err != nil {
+			t.Errorf("Test %d: failed to parse metadata: %v", i, err)
+			continue
+		}
+		if keyID != test.KeyID {
+			t.Errorf("Test %d: Key-ID mismatch: got '%s' - want '%s'", i, keyID, test.KeyID)
+		}
+		if !bytes.Equal(kmsKey, test.SealedDataKey) {
+			t.Errorf("Test %d: sealed KMS data mismatch: got '%v' - want '%v'", i, kmsKey, test.SealedDataKey)
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: seal algorithm mismatch: got '%s' - want '%s'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: IV mismatch: got '%v' - want '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: sealed key mismatch: got '%v' - want '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+	}
+
+	defer func() {
+		if err := recover(); err == nil || err != logger.ErrCritical {
+			t.Errorf("Expected '%s' panic for invalid seal algorithm but got '%s'", logger.ErrCritical, err)
+		}
+	}()
+	_ = S3.CreateMetadata(nil, "", []byte{}, SealedKey{Algorithm: InsecureSealAlgorithm})
+}
+
+var ssecCreateMetadataTests = []SealedKey{
+	{Algorithm: SealAlgorithm},
+	{IV: [32]byte{0xff}, Key: [64]byte{0x7e}, Algorithm: SealAlgorithm},
+}
+
+func TestSSECCreateMetadata(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+	for i, test := range s3CreateMetadataTests {
+		metadata := SSEC.CreateMetadata(nil, test.SealedKey)
+		sealedKey, err := SSEC.ParseMetadata(metadata)
+		if err != nil {
+			t.Errorf("Test %d: failed to parse metadata: %v", i, err)
+			continue
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: seal algorithm mismatch: got '%s' - want '%s'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: IV mismatch: got '%v' - want '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: sealed key mismatch: got '%v' - want '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+	}
+
+	defer func() {
+		if err := recover(); err == nil || err != logger.ErrCritical {
+			t.Errorf("Expected '%s' panic for invalid seal algorithm but got '%s'", logger.ErrCritical, err)
+		}
+	}()
+	_ = SSEC.CreateMetadata(nil, SealedKey{Algorithm: InsecureSealAlgorithm})
+}

cmd/crypto/sse.go

@@ -25,16 +25,46 @@ import (
 )
 
 const (
+	// SSEMultipart is the metadata key indicating that the object
+	// was uploaded using the S3 multipart API and stored using
+	// some from of server-side-encryption.
+	SSEMultipart = "X-Minio-Internal-Encrypted-Multipart"
+
+	// SSEIV is the metadata key referencing the random initialization
+	// vector (IV) used for SSE-S3 and SSE-C key derivation.
+	SSEIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
+
+	// SSESealAlgorithm is the metadata key referencing the algorithm
+	// used by SSE-C and SSE-S3 to encrypt the object.
+	SSESealAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
+
+	// SSECSealedKey is the metadata key referencing the sealed object-key for SSE-C.
+	SSECSealedKey = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
+
 	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
 	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+
 	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
 	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
 	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+
 	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
 	// by KMS. It is only used for SSE-S3 + KMS.
 	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
 )
 
+const (
+	// SealAlgorithm is the encryption/sealing algorithm used to derive & seal
+	// the key-encryption-key and to en/decrypt the object data.
+	SealAlgorithm = "DAREv2-HMAC-SHA256"
+
+	// InsecureSealAlgorithm is the legacy encryption/sealing algorithm used
+	// to derive & seal the key-encryption-key and to en/decrypt the object data.
+	// This algorithm should not be used for new objects because its key derivation
+	// is not optimal. See: https://github.com/minio/minio/pull/6121
+	InsecureSealAlgorithm = "DARE-SHA256"
+)
+
 // EncryptSinglePart encrypts an io.Reader which must be the
 // the body of a single-part PUT request.
 func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
@@ -45,6 +75,14 @@ func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
 	return r
 }
 
+// EncryptMultiPart encrypts an io.Reader which must be the body of
+// multi-part PUT request. It derives an unique encryption key from
+// the partID and the object key.
+func EncryptMultiPart(r io.Reader, partID int, key ObjectKey) io.Reader {
+	partKey := key.DerivePartKey(uint32(partID))
+	return EncryptSinglePart(r, ObjectKey(partKey))
+}
+
 // DecryptSinglePart decrypts an io.Writer which must an object
 // uploaded with the single-part PUT API. The offset and length
 // specify the requested range.

cmd/disk-cache-config.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"path/filepath"
 	"strings"
 
@@ -44,6 +45,15 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 	if err = json.Unmarshal(data, _cfg); err != nil {
 		return err
 	}
+
+	if _cfg.Expiry < 0 {
+		return errors.New("config expiry value should not be negative")
+	}
+
+	if _cfg.MaxUse < 0 {
+		return errors.New("config max use value should not be null or negative")
+	}
+
 	if _, err = parseCacheDrives(_cfg.Drives); err != nil {
 		return err
 	}

cmd/disk-cache.go

@@ -32,14 +32,10 @@ import (
 	"github.com/djherbis/atime"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/wildcard"
-
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/wildcard"
 )
 
-// list of all errors that can be ignored in tree walk operation in disk cache
-var cacheTreeWalkIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errVolumeNotFound, errFileNotFound)
-
 const (
 	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
 	cacheSizeMultiplier  = 100
@@ -291,40 +287,32 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - list of fsObjects
-func listDirCacheFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks []*cacheFSObjects) listDirFunc {
-	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string, err error) {
+func listDirCacheFactory(isLeaf isLeafFunc, disks []*cacheFSObjects) listDirFunc {
+	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string) {
 		var entries []string
 		for _, disk := range disks {
 			// ignore disk-caches that might be missing/offline
 			if disk == nil {
 				continue
 			}
-			fs := disk.FSObjects
-			entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 
-			// For any reason disk was deleted or goes offline, continue
-			// and list from other disks if possible.
+			fs := disk.FSObjects
+			var err error
+			entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 			if err != nil {
-				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
-					continue
-				}
-				return nil, err
+				continue
 			}
 
 			// Filter entries that have the prefix prefixEntry.
 			entries = filterMatchingPrefix(entries, prefixEntry)
 			dirs = append(dirs, entries...)
 		}
-		return dirs, nil
+		return dirs
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
-		var cacheEntries []string
-		cacheEntries, err = listCacheDirs(bucket, prefixDir, prefixEntry)
-		if err != nil {
-			return nil, false, err
-		}
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
+		cacheEntries := listCacheDirs(bucket, prefixDir, prefixEntry)
 		for _, entry := range cacheEntries {
 			// Find elements in entries which are not in mergedEntries
 			idx := sort.SearchStrings(mergedEntries, entry)
@@ -335,7 +323,7 @@ func listDirCacheFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks [
 			mergedEntries = append(mergedEntries, entry)
 			sort.Strings(mergedEntries)
 		}
-		return mergedEntries, false, nil
+		return mergedEntries, false
 	}
 	return listDir
 }
@@ -371,7 +359,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			return fs.isObjectDir(bucket, object)
 		}
 
-		listDir := listDirCacheFactory(isLeaf, cacheTreeWalkIgnoredErrs, c.cache.cfs)
+		listDir := listDirCacheFactory(isLeaf, c.cache.cfs)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 

cmd/format-fs.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -55,24 +56,22 @@ type formatFSVersionDetect struct {
 	} `json:"fs"`
 }
 
+// Generic structure to manage both v1 and v2 structures
+type formatFS struct {
+	formatMetaV1
+	FS interface{} `json:"fs"`
+}
+
 // Returns the latest "fs" format V1
 func newFormatFSV1() (format *formatFSV1) {
 	f := &formatFSV1{}
 	f.Version = formatMetaVersionV1
 	f.Format = formatBackendFS
+	f.ID = mustGetUUID()
 	f.FS.Version = formatFSVersionV1
 	return f
 }
 
-// Returns the latest "fs" format V2
-func newFormatFSV2() (format *formatFSV2) {
-	f := &formatFSV2{}
-	f.Version = formatMetaVersionV1
-	f.Format = formatBackendFS
-	f.FS.Version = formatFSVersionV2
-	return f
-}
-
 // Returns the field formatMetaV1.Format i.e the string "fs" which is never likely to change.
 // We do not use this function in XL to get the format as the file is not fcntl-locked on XL.
 func formatMetaGetFormatBackendFS(r io.ReadSeeker) (string, error) {
@@ -115,7 +114,16 @@ func formatFSMigrateV1ToV2(ctx context.Context, wlk *lock.LockedFile, fsPath str
 		return err
 	}
 
-	return jsonSave(wlk.File, newFormatFSV2())
+	formatV1 := formatFSV1{}
+	if err = jsonLoad(wlk, &formatV1); err != nil {
+		return err
+	}
+
+	formatV2 := formatFSV2{}
+	formatV2.formatMetaV1 = formatV1.formatMetaV1
+	formatV2.FS.Version = formatFSVersionV2
+
+	return jsonSave(wlk.File, formatV2)
 }
 
 // Migrate the "fs" backend.
@@ -179,6 +187,12 @@ func createFormatFS(ctx context.Context, fsFormatPath string) error {
 // migrate the backend when we are actively working on the backend.
 func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, err error) {
 	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
+
+	// Add a deployment ID, if it does not exist.
+	if err := formatFSFixDeploymentID(fsFormatPath); err != nil {
+		return nil, err
+	}
+
 	// Any read on format.json should be done with read-lock.
 	// Any write on format.json should be done with write-lock.
 	for {
@@ -235,7 +249,8 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			rlk.Close()
 			// Hold write lock during migration so that we do not disturb any
 			// minio processes running in parallel.
-			wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+			var wlk *lock.LockedFile
+			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
 			if err == lock.ErrAlreadyLocked {
 				// Lock already present, sleep and attempt again.
 				time.Sleep(100 * time.Millisecond)
@@ -253,7 +268,94 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			// Successfully migrated, now try to hold a read-lock on format.json
 			continue
 		}
-
+		var id string
+		if id, err = formatFSGetDeploymentID(rlk); err != nil {
+			rlk.Close()
+			return nil, err
+		}
+		logger.SetDeploymentID(id)
 		return rlk, nil
 	}
 }
+
+func formatFSGetDeploymentID(rlk *lock.RLockedFile) (id string, err error) {
+	format := &formatFS{}
+	if err := jsonLoad(rlk, format); err != nil {
+		return "", err
+	}
+	return format.ID, nil
+}
+
+// Generate a deployment ID if one does not exist already.
+func formatFSFixDeploymentID(fsFormatPath string) error {
+	rlk, err := lock.RLockedOpenFile(fsFormatPath)
+	if err == nil {
+		// format.json can be empty in a rare condition when another
+		// minio process just created the file but could not hold lock
+		// and write to it.
+		var fi os.FileInfo
+		fi, err = rlk.Stat()
+		if err != nil {
+			rlk.Close()
+			return err
+		}
+		if fi.Size() == 0 {
+			rlk.Close()
+			return nil
+		}
+	}
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	formatBackend, err := formatMetaGetFormatBackendFS(rlk)
+	if err != nil {
+		rlk.Close()
+		return err
+	}
+	if formatBackend != formatBackendFS {
+		rlk.Close()
+		return fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)
+	}
+
+	format := &formatFS{}
+	err = jsonLoad(rlk, format)
+	rlk.Close()
+	if err != nil {
+		return err
+	}
+
+	// Check if it needs to be updated
+	if format.ID != "" {
+		return nil
+	}
+
+	for {
+		wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+		if err == lock.ErrAlreadyLocked {
+			// Lock already present, sleep and attempt again.
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		if err != nil {
+			return err
+		}
+		defer wlk.Close()
+
+		err = jsonLoad(wlk, format)
+		if err != nil {
+			return err
+		}
+
+		// Check if it needs to be updated
+		if format.ID != "" {
+			return nil
+		}
+		format.ID = mustGetUUID()
+		return jsonSave(wlk, format)
+	}
+
+}

cmd/format-meta.go

@@ -49,4 +49,6 @@ type formatMetaV1 struct {
 	Version string `json:"version"`
 	// Format indicates the backend format type, supports two values 'xl' and 'fs'.
 	Format string `json:"format"`
+	// ID is the identifier for the minio deployment
+	ID string `json:"id"`
 }

cmd/format-xl.go

@@ -22,11 +22,13 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"sync"
 
 	"encoding/hex"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/logger"
 	sha256 "github.com/minio/sha256-simd"
 )
 
@@ -86,10 +88,11 @@ type formatXLV1 struct {
 // Represents the V2 backend disk structure version
 // under `.minio.sys` and actual data namespace.
 // formatXLV2 - structure holds format config version '2'.
+// The V2 format to support "large bucket" support where a bucket
+// can span multiple erasure sets.
 type formatXLV2 struct {
-	Version string `json:"version"`
-	Format  string `json:"format"`
-	XL      struct {
+	formatMetaV1
+	XL struct {
 		Version string `json:"version"` // Version of 'xl' format.
 		This    string `json:"this"`    // This field carries assigned disk uuid.
 		// Sets field carries the input disk order generated the first
@@ -107,9 +110,8 @@ type formatXLV2 struct {
 // In .minio.sys/multipart we have:
 // sha256(bucket/object)/uploadID/[xl.json, part.1, part.2 ....]
 type formatXLV3 struct {
-	Version string `json:"version"`
-	Format  string `json:"format"`
-	XL      struct {
+	formatMetaV1
+	XL struct {
 		Version string `json:"version"` // Version of 'xl' format.
 		This    string `json:"this"`    // This field carries assigned disk uuid.
 		// Sets field carries the input disk order generated the first
@@ -127,6 +129,7 @@ func newFormatXLV3(numSets int, setLen int) *formatXLV3 {
 	format := &formatXLV3{}
 	format.Version = formatMetaVersionV1
 	format.Format = formatBackendXL
+	format.ID = mustGetUUID()
 	format.XL.Version = formatXLVersionV3
 	format.XL.DistributionAlgo = formatXLVersionV2DistributionAlgo
 	format.XL.Sets = make([][]string, numSets)
@@ -443,6 +446,112 @@ func checkFormatXLValues(formats []*formatXLV3) error {
 	return nil
 }
 
+// Get Deployment ID for the XL sets from format.json.
+// This need not be in quorum. Even if one of the format.json
+// file has this value, we assume it is valid.
+// If more than one format.json's have different id, it is considered a corrupt
+// backend format.
+func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (string, error) {
+	var deploymentID string
+	for _, format := range formats {
+		if format == nil || format.ID == "" {
+			continue
+		}
+		if reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) {
+			// Found an ID in one of the format.json file
+			// Set deploymentID for the first time.
+			if deploymentID == "" {
+				deploymentID = format.ID
+			} else if deploymentID != format.ID {
+				// DeploymentID found earlier doesn't match with the
+				// current format.json's ID.
+				return "", errCorruptedFormat
+			}
+		}
+	}
+	return deploymentID, nil
+}
+
+// formatXLFixDeploymentID - Add deployment id if it is not present.
+func formatXLFixDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
+	// Acquire lock on format.json
+	mutex := newNSLock(globalIsDistXL)
+	formatLock := mutex.NewNSLock(minioMetaBucket, formatConfigFile)
+	if err = formatLock.GetLock(globalHealingTimeout); err != nil {
+		return err
+	}
+	defer formatLock.Unlock()
+
+	// Attempt to load all `format.json` from all disks.
+	var sErrs []error
+	formats, sErrs := loadFormatXLAll(storageDisks)
+	for i, sErr := range sErrs {
+		if _, ok := formatCriticalErrors[sErr]; ok {
+			return fmt.Errorf("Disk %s: %s", globalEndpoints[i], sErr)
+		}
+	}
+
+	for index := range formats {
+		// If the XL sets do not match, set those formats to nil,
+		// We do not have to update the ID on those format.json file.
+		if formats[index] != nil && !reflect.DeepEqual(formats[index].XL.Sets, refFormat.XL.Sets) {
+			formats[index] = nil
+		}
+	}
+	refFormat.ID, err = formatXLGetDeploymentID(refFormat, formats)
+	if err != nil {
+		return err
+	}
+
+	// If ID is set, then some other node got the lock
+	// before this node could and generated an ID
+	// for the deployment. No need to generate one.
+	if refFormat.ID != "" {
+		return nil
+	}
+
+	// ID is generated for the first time,
+	// We set the ID in all the formats and update.
+	refFormat.ID = mustGetUUID()
+	for _, format := range formats {
+		if format != nil {
+			format.ID = refFormat.ID
+		}
+	}
+	// Deployment ID needs to be set on all the disks.
+	// Save `format.json` across all disks.
+	return saveFormatXLAll(ctx, storageDisks, formats)
+
+}
+
+// Update only the valid local disks which have not been updated before.
+func formatXLFixLocalDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) error {
+	// If this server was down when the deploymentID was updated
+	// then we make sure that we update the local disks with the deploymentID.
+	for index, storageDisk := range storageDisks {
+		if globalEndpoints[index].IsLocal && storageDisk != nil && storageDisk.IsOnline() {
+			format, err := loadFormatXL(storageDisk)
+			if err != nil {
+				// Disk can be offline etc.
+				// ignore the errors seen here.
+				continue
+			}
+			if format.ID != "" {
+				continue
+			}
+			if !reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) {
+				continue
+			}
+			format.ID = refFormat.ID
+			if err := saveFormatXL(storageDisk, format); err != nil {
+				logger.LogIf(ctx, err)
+				return fmt.Errorf("Unable to save format.json, %s", err)
+			}
+		}
+	}
+	return nil
+}
+
 // Get backend XL format in quorum `format.json`.
 func getFormatXLInQuorum(formats []*formatXLV3) (*formatXLV3, error) {
 	formatHashes := make([]string, len(formats))

cmd/format-xl_test.go

@@ -345,8 +345,10 @@ func TestCheckFormatXLValue(t *testing.T) {
 		// Invalid XL format version "2".
 		{
 			&formatXLV3{
-				Version: "2",
-				Format:  "XL",
+				formatMetaV1: formatMetaV1{
+					Version: "2",
+					Format:  "XL",
+				},
 				XL: struct {
 					Version          string     `json:"version"`
 					This             string     `json:"this"`
@@ -361,8 +363,10 @@ func TestCheckFormatXLValue(t *testing.T) {
 		// Invalid XL format "Unknown".
 		{
 			&formatXLV3{
-				Version: "1",
-				Format:  "Unknown",
+				formatMetaV1: formatMetaV1{
+					Version: "1",
+					Format:  "Unknown",
+				},
 				XL: struct {
 					Version          string     `json:"version"`
 					This             string     `json:"this"`
@@ -377,8 +381,10 @@ func TestCheckFormatXLValue(t *testing.T) {
 		// Invalid XL format version "0".
 		{
 			&formatXLV3{
-				Version: "1",
-				Format:  "XL",
+				formatMetaV1: formatMetaV1{
+					Version: "1",
+					Format:  "XL",
+				},
 				XL: struct {
 					Version          string     `json:"version"`
 					This             string     `json:"this"`
@@ -466,6 +472,61 @@ func TestGetFormatXLInQuorumCheck(t *testing.T) {
 	}
 }
 
+// Tests formatXLGetDeploymentID()
+func TestGetXLID(t *testing.T) {
+	setCount := 2
+	disksPerSet := 8
+
+	format := newFormatXLV3(setCount, disksPerSet)
+	formats := make([]*formatXLV3, 16)
+
+	for i := 0; i < setCount; i++ {
+		for j := 0; j < disksPerSet; j++ {
+			newFormat := *format
+			newFormat.XL.This = format.XL.Sets[i][j]
+			formats[i*disksPerSet+j] = &newFormat
+		}
+	}
+
+	// Return a format from list of formats in quorum.
+	quorumFormat, err := getFormatXLInQuorum(formats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check if the reference format and input formats are same.
+	var id string
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
+		t.Fatal(err)
+	}
+
+	if id == "" {
+		t.Fatal("ID cannot be empty.")
+	}
+
+	formats[0] = nil
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
+		t.Fatal(err)
+	}
+	if id == "" {
+		t.Fatal("ID cannot be empty.")
+	}
+
+	formats[1].XL.Sets[0][0] = "bad-uuid"
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
+		t.Fatal(err)
+	}
+
+	if id == "" {
+		t.Fatal("ID cannot be empty.")
+	}
+
+	formats[2].ID = "bad-id"
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat {
+		t.Fatal("Unexpected Success")
+	}
+}
+
 // Initialize new format sets.
 func TestNewFormatSets(t *testing.T) {
 	setCount := 2

cmd/fs-v1-multipart.go

@@ -543,25 +543,11 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 		// All parts except the last part has to be atleast 5MB.
 		if !isMinAllowedPartSize(fi.Size()) {
-			err = PartTooSmall{
+			return oi, PartTooSmall{
 				PartNumber: part.PartNumber,
 				PartSize:   fi.Size(),
 				PartETag:   part.ETag,
 			}
-			logger.LogIf(ctx, err)
-			return oi, err
-		}
-
-		// TODO: Make necessary changes in future as explained in the below comment.
-		// All parts except the last part has to be of same size. We are introducing this
-		// check to see if any clients break. If clients do not break then we can optimize
-		// multipart PutObjectPart by writing the part at the right offset using pwrite()
-		// so that we don't need to do background append at all. i.e by the time we get
-		// CompleteMultipartUpload we already have the full file available which can be
-		// renamed to the main name-space.
-		if partSize != fi.Size() {
-			logger.LogIf(ctx, PartsSizeUnequal{})
-			return oi, PartsSizeUnequal{}
 		}
 	}
 

cmd/fs-v1.go

@@ -968,13 +968,15 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
 // is a leaf or non-leaf entry.
 func (fs *FSObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool, err error) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
+		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil {
-			return nil, false, err
+			logger.LogIf(context.Background(), err)
+			return
 		}
 		entries, delayIsLeaf = filterListEntries(bucket, prefixDir, entries, prefixEntry, isLeaf)
-		return entries, delayIsLeaf, nil
+		return entries, delayIsLeaf
 	}
 
 	// Return list factory instance.

cmd/gateway-main.go

@@ -159,6 +159,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Initialize gateway config.
 	initConfig()
 
+	// Load logger subsystem
+	loadLoggers()
+
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()

cmd/gateway/gcs/gateway-gcs.go

@@ -554,16 +554,16 @@ func isGCSMarker(marker string) bool {
 
 // ListObjects - lists all blobs in GCS bucket filtered by prefix
 func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (minio.ListObjectsInfo, error) {
+	if maxKeys == 0 {
+		return minio.ListObjectsInfo{}, nil
+	}
+
 	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
 		Delimiter: delimiter,
 		Prefix:    prefix,
 		Versions:  false,
 	})
 
-	isTruncated := false
-	nextMarker := ""
-	prefixes := []string{}
-
 	// To accommodate S3-compatible applications using
 	// ListObjectsV1 to use object keys as markers to control the
 	// listing of objects, we use the following encoding scheme to
@@ -574,83 +574,86 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
 	//   prefixing "{minio}" to the GCS continuation token,
 	//   e.g, "{minio}CgRvYmoz"
 	//
-	// - Application supplied markers are used as-is to list
-	//   object keys that appear after it in the lexicographical order.
+	// - Application supplied markers are transformed to a
+	//   GCS continuation token.
 
 	// If application is using GCS continuation token we should
 	// strip the gcsTokenPrefix we added.
-	gcsMarker := isGCSMarker(marker)
-	if gcsMarker {
-		it.PageInfo().Token = strings.TrimPrefix(marker, gcsTokenPrefix)
+	token := ""
+	if marker != "" {
+		if isGCSMarker(marker) {
+			token = strings.TrimPrefix(marker, gcsTokenPrefix)
+		} else {
+			token = toGCSPageToken(marker)
+		}
 	}
+	nextMarker := ""
 
-	it.PageInfo().MaxSize = maxKeys
+	var prefixes []string
+	var objects []minio.ObjectInfo
+	var nextPageToken string
+	var err error
 
-	objects := []minio.ObjectInfo{}
+	pager := iterator.NewPager(it, maxKeys, token)
 	for {
-		if len(objects) >= maxKeys {
-			// check if there is one next object and
-			// if that one next object is our hidden
-			// metadata folder, then just break
-			// otherwise we've truncated the output
-			attrs, _ := it.Next()
-			if attrs != nil && attrs.Prefix == minio.GatewayMinioSysTmp {
-				break
-			}
-
-			isTruncated = true
-			break
-		}
-
-		attrs, err := it.Next()
-		if err == iterator.Done {
-			break
-		}
+		gcsObjects := make([]*storage.ObjectAttrs, 0)
+		nextPageToken, err = pager.NextPage(&gcsObjects)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return minio.ListObjectsInfo{}, gcsToObjectError(err, bucket, prefix)
 		}
 
-		nextMarker = toGCSPageToken(attrs.Name)
+		for _, attrs := range gcsObjects {
 
-		if attrs.Prefix == minio.GatewayMinioSysTmp {
-			// We don't return our metadata prefix.
-			continue
-		}
-		if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
-			// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
-			// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
-			// which will be helpful to observe the "directory structure" for debugging purposes.
-			if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
-				strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
+			// Due to minio.GatewayMinioSysTmp keys being skipped, the number of objects + prefixes
+			// returned may not total maxKeys. This behavior is compatible with the S3 spec which
+			// allows the response to include less keys than maxKeys.
+			if attrs.Prefix == minio.GatewayMinioSysTmp {
+				// We don't return our metadata prefix.
 				continue
 			}
-		}
-		if attrs.Prefix != "" {
-			prefixes = append(prefixes, attrs.Prefix)
-			continue
-		}
-		if !gcsMarker && attrs.Name <= marker {
-			// if user supplied a marker don't append
-			// objects until we reach marker (and skip it).
-			continue
+			if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
+				// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
+				// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
+				// which will be helpful to observe the "directory structure" for debugging purposes.
+				if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
+					strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
+					continue
+				}
+			}
+
+			if attrs.Prefix != "" {
+				prefixes = append(prefixes, attrs.Prefix)
+			} else {
+				objects = append(objects, fromGCSAttrsToObjectInfo(attrs))
+			}
+
+			// The NextMarker property should only be set in the response if a delimiter is used
+			if delimiter != "" {
+				if attrs.Prefix > nextMarker {
+					nextMarker = attrs.Prefix
+				} else if attrs.Name > nextMarker {
+					nextMarker = attrs.Name
+				}
+			}
 		}
 
-		objects = append(objects, minio.ObjectInfo{
-			Name:            attrs.Name,
-			Bucket:          attrs.Bucket,
-			ModTime:         attrs.Updated,
-			Size:            attrs.Size,
-			ETag:            minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C)),
-			UserDefined:     attrs.Metadata,
-			ContentType:     attrs.ContentType,
-			ContentEncoding: attrs.ContentEncoding,
-		})
+		// Exit the loop if at least one item can be returned from
+		// the current page or there are no more pages available
+		if nextPageToken == "" || len(prefixes)+len(objects) > 0 {
+			break
+		}
+	}
+
+	if nextPageToken == "" {
+		nextMarker = ""
+	} else if nextMarker != "" {
+		nextMarker = gcsTokenPrefix + toGCSPageToken(nextMarker)
 	}
 
 	return minio.ListObjectsInfo{
-		IsTruncated: isTruncated,
-		NextMarker:  gcsTokenPrefix + nextMarker,
+		IsTruncated: nextPageToken != "",
+		NextMarker:  nextMarker,
 		Prefixes:    prefixes,
 		Objects:     objects,
 	}, nil
@@ -658,8 +661,8 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
 
 // ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix
 func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
-	if continuationToken == "" && startAfter != "" {
-		continuationToken = toGCSPageToken(startAfter)
+	if maxKeys == 0 {
+		return minio.ListObjectsV2Info{ContinuationToken: continuationToken}, nil
 	}
 
 	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
@@ -668,62 +671,62 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
 		Versions:  false,
 	})
 
-	isTruncated := false
-	it.PageInfo().MaxSize = maxKeys
-
-	if continuationToken != "" {
-		// If client sends continuationToken, set it
-		it.PageInfo().Token = continuationToken
-	} else {
-		// else set the continuationToken to return
-		continuationToken = it.PageInfo().Token
-		if continuationToken != "" {
-			// If GCS SDK sets continuationToken, it means there are more than maxKeys in the current page
-			// and the response will be truncated
-			isTruncated = true
-		}
+	token := continuationToken
+	if token == "" && startAfter != "" {
+		token = toGCSPageToken(startAfter)
 	}
 
 	var prefixes []string
 	var objects []minio.ObjectInfo
+	var nextPageToken string
+	var err error
 
+	pager := iterator.NewPager(it, maxKeys, token)
 	for {
-		attrs, err := it.Next()
-		if err == iterator.Done {
-			break
-		}
-
+		gcsObjects := make([]*storage.ObjectAttrs, 0)
+		nextPageToken, err = pager.NextPage(&gcsObjects)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return minio.ListObjectsV2Info{}, gcsToObjectError(err, bucket, prefix)
 		}
 
-		if attrs.Prefix == minio.GatewayMinioSysTmp {
-			// We don't return our metadata prefix.
-			continue
-		}
-		if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
-			// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
-			// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
-			// which will be helpful to observe the "directory structure" for debugging purposes.
-			if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
-				strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
+		for _, attrs := range gcsObjects {
+
+			// Due to minio.GatewayMinioSysTmp keys being skipped, the number of objects + prefixes
+			// returned may not total maxKeys. This behavior is compatible with the S3 spec which
+			// allows the response to include less keys than maxKeys.
+			if attrs.Prefix == minio.GatewayMinioSysTmp {
+				// We don't return our metadata prefix.
 				continue
 			}
+			if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
+				// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
+				// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
+				// which will be helpful to observe the "directory structure" for debugging purposes.
+				if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
+					strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
+					continue
+				}
+			}
+
+			if attrs.Prefix != "" {
+				prefixes = append(prefixes, attrs.Prefix)
+			} else {
+				objects = append(objects, fromGCSAttrsToObjectInfo(attrs))
+			}
 		}
 
-		if attrs.Prefix != "" {
-			prefixes = append(prefixes, attrs.Prefix)
-			continue
+		// Exit the loop if at least one item can be returned from
+		// the current page or there are no more pages available
+		if nextPageToken == "" || len(prefixes)+len(objects) > 0 {
+			break
 		}
-
-		objects = append(objects, fromGCSAttrsToObjectInfo(attrs))
 	}
 
 	return minio.ListObjectsV2Info{
-		IsTruncated:           isTruncated,
+		IsTruncated:           nextPageToken != "",
 		ContinuationToken:     continuationToken,
-		NextContinuationToken: continuationToken,
+		NextContinuationToken: nextPageToken,
 		Prefixes:              prefixes,
 		Objects:               objects,
 	}, nil

cmd/gateway/s3/gateway-s3.go

@@ -20,7 +20,9 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"math/rand"
 	"strings"
+	"time"
 
 	"github.com/minio/cli"
 	miniogo "github.com/minio/minio-go"
@@ -121,6 +123,31 @@ func (g *S3) Name() string {
 	return s3Backend
 }
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
+const (
+	letterIdxBits = 6                    // 6 bits to represent a letter index
+	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
+	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
+)
+
+// randString generates random names and prepends them with a known prefix.
+func randString(n int, src rand.Source, prefix string) string {
+	b := make([]byte, n)
+	// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
+	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
+		if remain == 0 {
+			cache, remain = src.Int63(), letterIdxMax
+		}
+		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
+			b[i] = letterBytes[idx]
+			i--
+		}
+		cache >>= letterIdxBits
+		remain--
+	}
+	return prefix + string(b[0:30-len(prefix)])
+}
+
 // newS3 - Initializes a new client by auto probing S3 server signature.
 func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 	if url == "" {
@@ -137,13 +164,13 @@ func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+	if _, err = clnt.BucketExists(probeBucketName); err != nil {
 		clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
 		if err != nil {
 			return nil, err
 		}
-		if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+		if _, err = clnt.BucketExists(probeBucketName); err != nil {
 			return nil, err
 		}
 	}

cmd/generic-handlers.go

@@ -723,6 +723,26 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	l.handler.ServeHTTP(w, r)
 }
 
+// requestIDHeaderHandler sets x-amz-request-id header.
+// Previously, this value was set right before a response
+// was sent to the client.So, logger and Error response XML
+// were not using this value.
+// This is set here so that this header can be logged as
+// part of the log entry and Error response XML.
+type requestIDHeaderHandler struct {
+	handler http.Handler
+}
+
+func addrequestIDHeader(h http.Handler) http.Handler {
+	return requestIDHeaderHandler{handler: h}
+}
+
+func (s requestIDHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Set unique request ID for each response.
+	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
+	s.handler.ServeHTTP(w, r)
+}
+
 type securityHeaderHandler struct {
 	handler http.Handler
 }

cmd/healthcheck-handler.go

@@ -17,10 +17,12 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"net/http"
+	"os"
 	"runtime"
+
+	"github.com/minio/minio/cmd/logger"
 )
 
 const (
@@ -40,19 +42,38 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
-// LivenessCheckHandler -- checks if server can ListBuckets internally. If not, server is
-// considered to have failed and needs to be restarted.
+// LivenessCheckHandler -- checks if server can reach its disks internally.
+// If not, server is considered to have failed and needs to be restarted.
 // Liveness probes are used to detect situations where application (minio)
 // has gone into a state where it can not recover except by being restarted.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "LivenessCheckHandler")
+
 	objLayer := newObjectLayerFn()
 	// Service not initialized yet
 	if objLayer == nil {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
-	// List buckets is unsuccessful, means server is having issues, send 503 service unavailable
-	if _, err := objLayer.ListBuckets(context.Background()); err != nil {
+	var totalLocalDisks int
+	var erroredDisks int
+	for _, endpoint := range globalEndpoints {
+		// Check only if local disks are accessible, we do not have
+		// to reach to rest of the other servers in a distributed setup.
+		if endpoint.IsLocal {
+			totalLocalDisks++
+			// Attempt a stat to backend, any error resulting
+			// from this Stat() operation is considered as backend
+			// is not available, count them as errors.
+			if _, err := os.Stat(endpoint.Path); err != nil {
+				logger.LogIf(ctx, err)
+				erroredDisks++
+			}
+		}
+	}
+	// If all exported local disks have errored, we simply let kubernetes
+	// take us down.
+	if totalLocalDisks == erroredDisks {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}

cmd/logger/console.go

@@ -0,0 +1,101 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// ConsoleTarget implements loggerTarget to send log
+// in plain or json format to the standard output.
+type ConsoleTarget struct{}
+
+func (c *ConsoleTarget) send(entry logEntry) error {
+	if jsonFlag {
+		logJSON, err := json.Marshal(&entry)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(logJSON))
+		return nil
+	}
+
+	trace := make([]string, len(entry.Trace.Source))
+
+	// Add a sequence number and formatting for each stack trace
+	// No formatting is required for the first entry
+	for i, element := range entry.Trace.Source {
+		trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
+	}
+
+	tagString := ""
+	for key, value := range entry.Trace.Variables {
+		if value != "" {
+			if tagString != "" {
+				tagString += ", "
+			}
+			tagString += key + "=" + value
+		}
+	}
+
+	apiString := "API: " + entry.API.Name + "("
+	if entry.API.Args != nil && entry.API.Args.Bucket != "" {
+		apiString = apiString + "bucket=" + entry.API.Args.Bucket
+	}
+	if entry.API.Args != nil && entry.API.Args.Object != "" {
+		apiString = apiString + ", object=" + entry.API.Args.Object
+	}
+	apiString += ")"
+	timeString := "Time: " + time.Now().Format(loggerTimeFormat)
+
+	var requestID string
+	if entry.RequestID != "" {
+		requestID = "\nRequestID: " + entry.RequestID
+	}
+
+	var remoteHost string
+	if entry.RemoteHost != "" {
+		remoteHost = "\nRemoteHost: " + entry.RemoteHost
+	}
+
+	var userAgent string
+	if entry.UserAgent != "" {
+		userAgent = "\nUserAgent: " + entry.UserAgent
+	}
+
+	if len(entry.Trace.Variables) > 0 {
+		tagString = "\n       " + tagString
+	}
+
+	var msg = colorFgRed(colorBold(entry.Trace.Message))
+	var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
+		apiString, timeString, requestID, remoteHost, userAgent,
+		msg, tagString, strings.Join(trace, "\n"))
+
+	fmt.Println(output)
+	return nil
+}
+
+// NewConsole initializes a new logger target
+// which prints log directly in the standard
+// output.
+func NewConsole() LoggingTarget {
+	return &ConsoleTarget{}
+}

cmd/logger/http.go

@@ -0,0 +1,88 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"net/http"
+)
+
+// HTTPTarget implements loggerTarget and sends the json
+// format of a log entry to the configured http endpoint.
+// An internal buffer of logs is maintained but when the
+// buffer is full, new logs are just ignored and an error
+// is returned to the caller.
+type HTTPTarget struct {
+	// Channel of log entries
+	logCh chan logEntry
+	// HTTP(s) endpoint
+	endpoint string
+	client   http.Client
+}
+
+func (h *HTTPTarget) startHTTPLogger() {
+	// Create a routine which sends json logs received
+	// from an internal channel.
+	go func() {
+		for entry := range h.logCh {
+			logJSON, err := json.Marshal(&entry)
+			if err != nil {
+				continue
+			}
+
+			req, err := http.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
+			req.Header.Set("Content-Type", "application/json")
+
+			resp, err := h.client.Do(req)
+			if err != nil {
+				continue
+			}
+			if resp.Body != nil {
+				resp.Body.Close()
+			}
+		}
+	}()
+}
+
+// NewHTTP initializes a new logger target which
+// sends log over http to the specified endpoint
+func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
+	h := HTTPTarget{
+		endpoint: endpoint,
+		client: http.Client{
+			Transport: transport,
+		},
+		logCh: make(chan logEntry, 10000),
+	}
+
+	h.startHTTPLogger()
+	return &h
+}
+
+func (h *HTTPTarget) send(entry logEntry) error {
+	select {
+	case h.logCh <- entry:
+	default:
+		// log channel is full, do not wait and return
+		// an error immediately to the caller
+		return errors.New("log buffer full")
+	}
+
+	return nil
+}

cmd/logger/logger.go

@@ -28,7 +28,6 @@ import (
 	"time"
 
 	c "github.com/minio/mc/pkg/console"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 // Disable disables all logging, false by default. (used for "go test")
@@ -122,14 +121,15 @@ type api struct {
 }
 
 type logEntry struct {
-	Level      string      `json:"level"`
-	Time       string      `json:"time"`
-	API        *api        `json:"api,omitempty"`
-	RemoteHost string      `json:"remotehost,omitempty"`
-	RequestID  string      `json:"requestID,omitempty"`
-	UserAgent  string      `json:"userAgent,omitempty"`
-	Message    string      `json:"message,omitempty"`
-	Trace      *traceEntry `json:"error,omitempty"`
+	DeploymentID string      `json:"deploymentid,omitempty"`
+	Level        string      `json:"level"`
+	Time         string      `json:"time"`
+	API          *api        `json:"api,omitempty"`
+	RemoteHost   string      `json:"remotehost,omitempty"`
+	RequestID    string      `json:"requestID,omitempty"`
+	UserAgent    string      `json:"userAgent,omitempty"`
+	Message      string      `json:"message,omitempty"`
+	Trace        *traceEntry `json:"error,omitempty"`
 }
 
 // quiet: Hide startup messages if enabled
@@ -138,8 +138,15 @@ var (
 	quiet, jsonFlag bool
 	// Custom function to format error
 	errorFmtFunc func(string, error, bool) string
+
+	deploymentID string
 )
 
+// SetDeploymentID - Used to set the deployment ID, in XL and FS mode
+func SetDeploymentID(id string) {
+	deploymentID = id
+}
+
 // EnableQuiet - turns quiet option on.
 func EnableQuiet() {
 	quiet = true
@@ -292,74 +299,21 @@ func LogIf(ctx context.Context, err error) {
 	// Get the cause for the Error
 	message := err.Error()
 
-	// Output the formatted log message at console
-	var output string
-	if jsonFlag {
-		logJSON, err := json.Marshal(&logEntry{
-			Level:      ErrorLvl.String(),
-			RemoteHost: req.RemoteHost,
-			RequestID:  req.RequestID,
-			UserAgent:  req.UserAgent,
-			Time:       time.Now().UTC().Format(time.RFC3339Nano),
-			API:        &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
-			Trace:      &traceEntry{Message: message, Source: trace, Variables: tags},
-		})
-		if err != nil {
-			panic(err)
-		}
-		output = string(logJSON)
-	} else {
-		// Add a sequence number and formatting for each stack trace
-		// No formatting is required for the first entry
-		for i, element := range trace {
-			trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
-		}
-
-		tagString := ""
-		for key, value := range tags {
-			if value != "" {
-				if tagString != "" {
-					tagString += ", "
-				}
-				tagString += key + "=" + value
-			}
-		}
-
-		apiString := "API: " + API + "("
-		if req.BucketName != "" {
-			apiString = apiString + "bucket=" + req.BucketName
-		}
-		if req.ObjectName != "" {
-			apiString = apiString + ", object=" + req.ObjectName
-		}
-		apiString += ")"
-		timeString := "Time: " + time.Now().Format(loggerTimeFormat)
-
-		var requestID string
-		if req.RequestID != "" {
-			requestID = "\nRequestID: " + req.RequestID
-		}
-
-		var remoteHost string
-		if req.RemoteHost != "" {
-			remoteHost = "\nRemoteHost: " + req.RemoteHost
-		}
-
-		var userAgent string
-		if req.UserAgent != "" {
-			userAgent = "\nUserAgent: " + req.UserAgent
-		}
-
-		if len(tags) > 0 {
-			tagString = "\n       " + tagString
-		}
-
-		var msg = colorFgRed(colorBold(message))
-		output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
-			apiString, timeString, requestID, remoteHost, userAgent,
-			msg, tagString, strings.Join(trace, "\n"))
+	entry := logEntry{
+		DeploymentID: deploymentID,
+		Level:        ErrorLvl.String(),
+		RemoteHost:   req.RemoteHost,
+		RequestID:    req.RequestID,
+		UserAgent:    req.UserAgent,
+		Time:         time.Now().UTC().Format(time.RFC3339Nano),
+		API:          &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
+		Trace:        &traceEntry{Message: message, Source: trace, Variables: tags},
+	}
+
+	// Iterate over all logger targets to send the log entry
+	for _, t := range Targets {
+		t.send(entry)
 	}
-	fmt.Println(output)
 }
 
 // ErrCritical is the value panic'd whenever CriticalIf is called.
@@ -423,23 +377,15 @@ func (f fatalMsg) quiet(msg string, args ...interface{}) {
 }
 
 var (
-	logTag       = "ERROR"
-	logBanner    = colorBgRed(colorFgWhite(colorBold(logTag))) + " "
-	emptyBanner  = colorBgRed(strings.Repeat(" ", len(logTag))) + " "
-	minimumWidth = 80
-	bannerWidth  = len(logTag) + 1
+	logTag      = "ERROR"
+	logBanner   = colorBgRed(colorFgWhite(colorBold(logTag))) + " "
+	emptyBanner = colorBgRed(strings.Repeat(" ", len(logTag))) + " "
+	bannerWidth = len(logTag) + 1
 )
 
 func (f fatalMsg) pretty(msg string, args ...interface{}) {
 	// Build the passed error message
 	errMsg := fmt.Sprintf(msg, args...)
-	// Check terminal width
-	termWidth, _, err := terminal.GetSize(0)
-	if err != nil || termWidth < minimumWidth {
-		termWidth = minimumWidth
-	}
-	// Calculate available widht without the banner
-	width := termWidth - bannerWidth
 
 	tagPrinted := false
 
@@ -470,13 +416,8 @@ func (f fatalMsg) pretty(msg string, args ...interface{}) {
 			ansiRestoreAttributes()
 			ansiMoveRight(bannerWidth)
 			// Continue  error message printing
-			if len(line) > width {
-				fmt.Println(line[:width])
-				line = line[width:]
-			} else {
-				fmt.Println(line)
-				break
-			}
+			fmt.Println(line)
+			break
 		}
 	}
 

cmd/logger/targets.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,9 +14,20 @@
  * limitations under the License.
  */
 
-package madmin
+package logger
 
-const (
-	// Unsigned payload.
-	unsignedPayload = "UNSIGNED-PAYLOAD"
-)
+// LoggingTarget is the entity that we will receive
+// a single log entry and send it to the log target
+//   e.g. send the log to a http server
+type LoggingTarget interface {
+	send(entry logEntry) error
+}
+
+// Targets is the set of enabled loggers
+var Targets = []LoggingTarget{}
+
+// AddTarget adds a new logger target to the
+// list of enabled loggers
+func AddTarget(t LoggingTarget) {
+	Targets = append(Targets, t)
+}

cmd/object-api-errors.go

@@ -320,13 +320,6 @@ func (e InvalidPart) Error() string {
 	return "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag."
 }
 
-// PartsSizeUnequal - All parts except the last part should be of the same size
-type PartsSizeUnequal struct{}
-
-func (e PartsSizeUnequal) Error() string {
-	return "All parts except the last part should be of the same size"
-}
-
 // PartTooSmall - error if part size is less than 5MB.
 type PartTooSmall struct {
 	PartSize   int64

cmd/object-handlers.go

@@ -68,7 +68,7 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
 // This implementation of the GET operation retrieves object. To use GET,
 // you must have READ access to the object.
 func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetObject")
+	ctx := newContext(r, w, "GetObject")
 
 	var object, bucket string
 	vars := mux.Vars(r)
@@ -218,7 +218,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 // -----------
 // The HEAD operation retrieves metadata from an object without returning the object itself.
 func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "HeadObject")
+	ctx := newContext(r, w, "HeadObject")
 
 	var object, bucket string
 	vars := mux.Vars(r)
@@ -341,7 +341,7 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
 // This implementation of the PUT operation adds an object to a bucket
 // while reading the object from another source.
 func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "CopyObject")
+	ctx := newContext(r, w, "CopyObject")
 
 	vars := mux.Vars(r)
 	dstBucket := vars["bucket"]
@@ -638,7 +638,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 // ----------
 // This implementation of the PUT operation adds an object to a bucket.
 func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutObject")
+	ctx := newContext(r, w, "PutObject")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -845,7 +845,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 // NewMultipartUploadHandler - New multipart upload.
 func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "NewMultipartUpload")
+	ctx := newContext(r, w, "NewMultipartUpload")
 
 	var object, bucket string
 	vars := mux.Vars(r)
@@ -932,7 +932,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 
 // CopyObjectPartHandler - uploads a part by copying data from an existing object as data source.
 func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "CopyObjectPart")
+	ctx := newContext(r, w, "CopyObjectPart")
 
 	vars := mux.Vars(r)
 	dstBucket := vars["bucket"]
@@ -1131,7 +1131,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 // PutObjectPartHandler - uploads an incoming part for an ongoing multipart operation.
 func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutObjectPart")
+	ctx := newContext(r, w, "PutObjectPart")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1330,7 +1330,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 // AbortMultipartUploadHandler - Abort multipart upload
 func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "AbortMultipartUpload")
+	ctx := newContext(r, w, "AbortMultipartUpload")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1369,7 +1369,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 
 // ListObjectPartsHandler - List object parts
 func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectParts")
+	ctx := newContext(r, w, "ListObjectParts")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1409,7 +1409,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 
 // CompleteMultipartUploadHandler - Complete multipart upload.
 func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "CompleteMultipartUpload")
+	ctx := newContext(r, w, "CompleteMultipartUpload")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1518,7 +1518,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 // DeleteObjectHandler - delete an object
 func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteObject")
+	ctx := newContext(r, w, "DeleteObject")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/object-handlers_test.go

@@ -263,7 +263,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey), getGetObjectURL("", bucketName, "abcd"))),
+			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey), getGetObjectURL("", bucketName, "abcd"), "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 3.
@@ -287,7 +287,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidRange), getGetObjectURL("", bucketName, objectName))),
+			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidRange), getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusRequestedRangeNotSatisfiable,
 		},
 		// Test case - 5.
@@ -313,7 +313,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  "Invalid-AccessID",
 			secretKey:  credentials.SecretKey,
 
-			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID), getGetObjectURL("", bucketName, objectName))),
+			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID), getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusForbidden,
 		},
 		// Test case - 7.
@@ -326,7 +326,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			secretKey:  credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidObjectName),
-				getGetObjectURL("", bucketName, "../../etc"))),
+				getGetObjectURL("", bucketName, "../../etc"), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 8.
@@ -339,7 +339,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			secretKey:  credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey),
-				"/"+bucketName+"/"+". ./. ./etc")),
+				"/"+bucketName+"/"+". ./. ./etc", "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 9.
@@ -352,7 +352,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			secretKey:  credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidObjectName),
-				"/"+bucketName+"/"+". ./../etc")),
+				"/"+bucketName+"/"+". ./../etc", "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 10.
@@ -365,7 +365,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			secretKey:  credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey),
-				getGetObjectURL("", bucketName, "etc/path/proper/.../etc"))),
+				getGetObjectURL("", bucketName, "etc/path/proper/.../etc"), "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 	}
@@ -2238,7 +2238,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidPart{})),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 2.
@@ -2253,7 +2253,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrMalformedXML),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 3.
@@ -2268,7 +2268,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidUploadID{UploadID: "abc"})),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 4.
@@ -2283,7 +2283,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 
 			expectedContent: encodeResponse(completeMultipartAPIError{int64(4), int64(5242880), 1, "e2fc714c4727ee9395f324cd2e7f331f",
 				getAPIErrorResponse(getAPIError(toAPIErrorCode(PartTooSmall{PartNumber: 1})),
-					getGetObjectURL("", bucketName, objectName))}),
+					getGetObjectURL("", bucketName, objectName), "")}),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 5.
@@ -2297,7 +2297,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidPart{})),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 6.
@@ -2312,7 +2312,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidPartOrder),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 7.
@@ -2327,7 +2327,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusForbidden,
 		},
 		// Test case - 8.

cmd/posix-errors.go

@@ -24,22 +24,46 @@ import (
 
 // Function not implemented error
 func isSysErrNoSys(err error) bool {
-	return err == syscall.ENOSYS
+	if pathErr, ok := err.(*os.PathError); ok {
+		switch pathErr.Err {
+		case syscall.ENOSYS:
+			return true
+		}
+	}
+	return false
 }
 
 // Not supported error
 func isSysErrOpNotSupported(err error) bool {
-	return err == syscall.EOPNOTSUPP
+	if pathErr, ok := err.(*os.PathError); ok {
+		switch pathErr.Err {
+		case syscall.EOPNOTSUPP:
+			return true
+		}
+	}
+	return false
 }
 
 // No space left on device error
 func isSysErrNoSpace(err error) bool {
-	return err == syscall.ENOSPC
+	if pathErr, ok := err.(*os.PathError); ok {
+		switch pathErr.Err {
+		case syscall.ENOSPC:
+			return true
+		}
+	}
+	return false
 }
 
 // Input/output error
 func isSysErrIO(err error) bool {
-	return err == syscall.EIO
+	if pathErr, ok := err.(*os.PathError); ok {
+		switch pathErr.Err {
+		case syscall.EIO:
+			return true
+		}
+	}
+	return false
 }
 
 // Check if the given error corresponds to EISDIR (is a directory).

cmd/posix-list-dir-nix.go

@@ -22,7 +22,6 @@ import (
 	"os"
 	"path"
 	"runtime"
-	"strings"
 	"sync"
 	"syscall"
 	"unsafe"
@@ -124,18 +123,12 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 
 	d, err := os.Open(dirPath)
 	if err != nil {
-		// File is really not found.
-		if os.IsNotExist(err) {
+		if os.IsNotExist(err) || isSysErrNotDir(err) {
 			return nil, errFileNotFound
 		}
 		if os.IsPermission(err) {
 			return nil, errFileAccessDenied
 		}
-
-		// File path cannot be verified since one of the parents is a file.
-		if strings.Contains(err.Error(), "not a directory") {
-			return nil, errFileNotFound
-		}
 		return nil, err
 	}
 	defer d.Close()
@@ -148,6 +141,9 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 	for !done {
 		nbuf, err := syscall.ReadDirent(fd, buf)
 		if err != nil {
+			if isSysErrNotDir(err) {
+				return nil, errFileNotFound
+			}
 			return nil, err
 		}
 		if nbuf <= 0 {

cmd/posix.go

@@ -327,15 +327,18 @@ func (s *posix) checkDiskFound() (err error) {
 	if !s.IsOnline() {
 		return errDiskNotFound
 	}
-	_, err = os.Stat((s.diskPath))
+	_, err = os.Stat(s.diskPath)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errDiskNotFound
 		} else if isSysErrTooLong(err) {
 			return errFileNameTooLong
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
+		return err
 	}
-	return err
+	return nil
 }
 
 // diskUsage returns du information for the posix path, in a continuous routine.
@@ -421,7 +424,7 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 // Make a volume entry.
 func (s *posix) MakeVol(volume string) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -451,6 +454,8 @@ func (s *posix) MakeVol(volume string) (err error) {
 		}
 		if os.IsPermission(err) {
 			return errDiskAccessDenied
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 		return err
 	}
@@ -462,7 +467,7 @@ func (s *posix) MakeVol(volume string) (err error) {
 // ListVols - list volumes.
 func (s *posix) ListVols() (volsInfo []VolInfo, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -475,8 +480,11 @@ func (s *posix) ListVols() (volsInfo []VolInfo, err error) {
 		return nil, err
 	}
 
-	volsInfo, err = listVols((s.diskPath))
+	volsInfo, err = listVols(s.diskPath)
 	if err != nil {
+		if isSysErrIO(err) {
+			return nil, errFaultyDisk
+		}
 		return nil, err
 	}
 	for i, vol := range volsInfo {
@@ -505,11 +513,13 @@ func listVols(dirPath string) ([]VolInfo, error) {
 			continue
 		}
 		var fi os.FileInfo
-		fi, err = os.Stat((pathJoin(dirPath, entry)))
+		fi, err = os.Stat(pathJoin(dirPath, entry))
 		if err != nil {
 			// If the file does not exist, skip the entry.
 			if os.IsNotExist(err) {
 				continue
+			} else if isSysErrIO(err) {
+				return nil, errFaultyDisk
 			}
 			return nil, err
 		}
@@ -526,7 +536,7 @@ func listVols(dirPath string) ([]VolInfo, error) {
 // StatVol - get volume info.
 func (s *posix) StatVol(volume string) (volInfo VolInfo, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -550,6 +560,8 @@ func (s *posix) StatVol(volume string) (volInfo VolInfo, err error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			return VolInfo{}, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return VolInfo{}, errFaultyDisk
 		}
 		return VolInfo{}, err
 	}
@@ -565,7 +577,7 @@ func (s *posix) StatVol(volume string) (volInfo VolInfo, err error) {
 // DeleteVol - delete a volume.
 func (s *posix) DeleteVol(volume string) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -591,8 +603,9 @@ func (s *posix) DeleteVol(volume string) (err error) {
 			return errVolumeNotEmpty
 		} else if os.IsPermission(err) {
 			return errDiskAccessDenied
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
-
 		return err
 	}
 	return nil
@@ -602,7 +615,7 @@ func (s *posix) DeleteVol(volume string) (err error) {
 // If an entry is a directory it will be returned with a trailing "/".
 func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -625,6 +638,8 @@ func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, er
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -644,7 +659,7 @@ func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, er
 // not use this on large files as it would cause server to crash.
 func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -666,6 +681,8 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -694,6 +711,8 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 				}
 			}
 			return nil, pathErr
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -715,7 +734,7 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 // semantics are same as io.ReadFull.
 func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (n int64, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -737,6 +756,8 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 	if err != nil {
 		if os.IsNotExist(err) {
 			return 0, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return 0, errFaultyDisk
 		}
 		return 0, err
 	}
@@ -756,6 +777,8 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 			return 0, errFileAccessDenied
 		} else if isSysErrNotDir(err) {
 			return 0, errFileAccessDenied
+		} else if isSysErrIO(err) {
+			return 0, errFaultyDisk
 		}
 		return 0, err
 	}
@@ -807,7 +830,7 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 
 func (s *posix) createFile(volume, path string) (f *os.File, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -829,6 +852,8 @@ func (s *posix) createFile(volume, path string) (f *os.File, err error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -859,6 +884,8 @@ func (s *posix) createFile(volume, path string) (f *os.File, err error) {
 			return nil, errFileAccessDenied
 		} else if os.IsPermission(err) {
 			return nil, errFileAccessDenied
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -875,7 +902,7 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 	}
 
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -886,6 +913,9 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 
 	// Validate if disk is indeed free.
 	if err = checkDiskFree(s.diskPath, fileSize); err != nil {
+		if isSysErrIO(err) {
+			return errFaultyDisk
+		}
 		return err
 	}
 
@@ -907,7 +937,7 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 		case isSysErrNoSpace(e):
 			err = errDiskFull
 		case isSysErrIO(e):
-			err = e
+			err = errFaultyDisk
 		default:
 			// For errors: EBADF, EINTR, EINVAL, ENODEV, EPERM, ESPIPE  and ETXTBSY
 			// Appending was failed anyway, returns unexpected error
@@ -922,7 +952,7 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 // path this call explicitly creates it.
 func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -944,7 +974,7 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 // StatFile - get file info.
 func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -979,10 +1009,10 @@ func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
 		// File is really not found.
 		if os.IsNotExist(err) {
 			return FileInfo{}, errFileNotFound
-		}
-
-		// File path cannot be verified since one of the parents is a file.
-		if isSysErrNotDir(err) {
+		} else if isSysErrIO(err) {
+			return FileInfo{}, errFaultyDisk
+		} else if isSysErrNotDir(err) {
+			// File path cannot be verified since one of the parents is a file.
 			return FileInfo{}, errFileNotFound
 		}
 
@@ -1023,6 +1053,8 @@ func deleteFile(basePath, deletePath string) error {
 			return errFileNotFound
 		} else if os.IsPermission(err) {
 			return errFileAccessDenied
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 		return err
 	}
@@ -1041,7 +1073,7 @@ func deleteFile(basePath, deletePath string) error {
 // DeleteFile - delete a file at path.
 func (s *posix) DeleteFile(volume, path string) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -1063,6 +1095,8 @@ func (s *posix) DeleteFile(volume, path string) (err error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 		return err
 	}
@@ -1081,7 +1115,7 @@ func (s *posix) DeleteFile(volume, path string) (err error) {
 // RenameFile - rename source path to destination path atomically.
 func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -1107,6 +1141,8 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 		return err
 	}
@@ -1114,6 +1150,8 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 	}
 
@@ -1136,7 +1174,9 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 		// we still need to allow overwriting an empty directory since it represents
 		// an object empty directory.
 		_, err = os.Stat(dstFilePath)
-
+		if isSysErrIO(err) {
+			return errFaultyDisk
+		}
 		if err == nil && !isDirEmpty(dstFilePath) {
 			return errFileAccessDenied
 		}
@@ -1146,6 +1186,9 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 	}
 
 	if err = renameAll(srcFilePath, dstFilePath); err != nil {
+		if isSysErrIO(err) {
+			return errFaultyDisk
+		}
 		return err
 	}
 

cmd/prepare-storage.go

@@ -186,6 +186,23 @@ func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, dr
 		return nil, err
 	}
 
+	// Get the deploymentID if set.
+	format.ID, err = formatXLGetDeploymentID(format, formatConfigs)
+	if err != nil {
+		return nil, err
+	}
+
+	if format.ID == "" {
+		if err = formatXLFixDeploymentID(context.Background(), storageDisks, format); err != nil {
+			return nil, err
+		}
+	}
+
+	logger.SetDeploymentID(format.ID)
+
+	if err = formatXLFixLocalDeploymentID(context.Background(), storageDisks, format); err != nil {
+		return nil, err
+	}
 	return format, nil
 }
 

cmd/routers.go

@@ -47,6 +47,8 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 
 // List of some generic handlers which are applied for all incoming requests.
 var globalHandlers = []HandlerFunc{
+	// set x-amz-request-id header.
+	addrequestIDHeader,
 	// set HTTP security headers such as Content-Security-Policy.
 	addSecurityHeaders,
 	// Forward path style requests to actual host in a bucket federated setup.

cmd/server-main.go

@@ -72,9 +72,6 @@ ENVIRONMENT VARIABLES:
      MINIO_ACCESS_KEY: Custom username or access key of minimum 3 characters in length.
      MINIO_SECRET_KEY: Custom password or secret key of minimum 8 characters in length.
 
-  ENDPOINTS:
-     MINIO_ENDPOINTS: List of all endpoints delimited by ' '.
-
   BROWSER:
      MINIO_BROWSER: To disable web browser access, set this value to "off".
 
@@ -106,9 +103,8 @@ EXAMPLES:
      $ export MINIO_DOMAIN=mydomain.com
      $ {{.HelpName}} --address mydomain.com:9000 /mnt/export
 
-  4. Start minio server on 64 disks server with endpoints through environment variable.
-     $ export MINIO_ENDPOINTS=/mnt/export{1...64}
-     $ {{.HelpName}}
+  4. Start erasure coded minio server on a node with 64 drives.
+     $ {{.HelpName}} /mnt/export{1...64}
 
   5. Start distributed minio server on an 8 node setup with 8 drives each. Run following command on all the 8 nodes.
      $ export MINIO_ACCESS_KEY=minio
@@ -221,6 +217,9 @@ func serverMain(ctx *cli.Context) {
 	// Initialize server config.
 	initConfig()
 
+	// Load logger subsystem
+	loadLoggers()
+
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()

cmd/test-utils_test.go

@@ -1855,7 +1855,7 @@ func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketN
 	}
 
 	// expected error response in bytes when objectLayer is not initialized, or set to `nil`.
-	expectedErrResponse := encodeResponse(getAPIErrorResponse(getAPIError(ErrAccessDenied), getGetObjectURL("", bucketName, objectName)))
+	expectedErrResponse := encodeResponse(getAPIErrorResponse(getAPIError(ErrAccessDenied), getGetObjectURL("", bucketName, objectName), ""))
 
 	// HEAD HTTTP request doesn't contain response body.
 	if anonReq.Method != "HEAD" {
@@ -1958,7 +1958,7 @@ func ExecObjectLayerAPINilTest(t TestErrHandler, bucketName, objectName, instanc
 	}
 	// expected error response in bytes when objectLayer is not initialized, or set to `nil`.
 	expectedErrResponse := encodeResponse(getAPIErrorResponse(getAPIError(ErrServerNotInitialized),
-		getGetObjectURL("", bucketName, objectName)))
+		getGetObjectURL("", bucketName, objectName), ""))
 
 	// HEAD HTTP Request doesn't contain body in its response,
 	// for other type of HTTP requests compare the response body content with the expected one.

cmd/tree-walk.go

@@ -86,7 +86,7 @@ func filterMatchingPrefix(entries []string, prefixEntry string) []string {
 }
 
 // "listDir" function of type listDirFunc returned by listDirFactory() - explained below.
-type listDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool, err error)
+type listDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool)
 
 // A function isLeaf of type isLeafFunc is used to detect if an entry is a leaf entry. There are four scenarios
 // where isLeaf should behave differently:
@@ -141,16 +141,8 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 			markerBase = markerSplit[1]
 		}
 	}
-	entries, delayIsLeaf, err := listDir(bucket, prefixDir, entryPrefixMatch)
-	if err != nil {
-		select {
-		case <-endWalkCh:
-			return errWalkAbort
-		case resultCh <- treeWalkResult{err: err}:
-			return err
-		}
-	}
 
+	entries, delayIsLeaf := listDir(bucket, prefixDir, entryPrefixMatch)
 	// When isleaf check is delayed, make sure that it is set correctly here.
 	if delayIsLeaf && isLeaf == nil {
 		return errInvalidArgument

cmd/tree-walk_test.go

@@ -196,7 +196,7 @@ func TestTreeWalk(t *testing.T) {
 		return len(entries) == 0
 	}
 
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk)
+	listDir := listDirFactory(context.Background(), isLeaf, disk)
 	// Simple test for prefix based walk.
 	testTreeWalkPrefix(t, listDir, isLeaf, isLeafDir)
 	// Simple test when marker is set.
@@ -240,7 +240,7 @@ func TestTreeWalkTimeout(t *testing.T) {
 		return len(entries) == 0
 	}
 
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk)
+	listDir := listDirFactory(context.Background(), isLeaf, disk)
 
 	// TreeWalk pool with 2 seconds timeout for tree-walk go routines.
 	pool := newTreeWalkPool(2 * time.Second)
@@ -315,7 +315,7 @@ func TestListDir(t *testing.T) {
 	// create listDir function.
 	listDir := listDirFactory(context.Background(), func(volume, prefix string) bool {
 		return !hasSuffix(prefix, slashSeparator)
-	}, xlTreeWalkIgnoredErrs, disk1, disk2)
+	}, disk1, disk2)
 
 	// Create file1 in fsDir1 and file2 in fsDir2.
 	disks := []StorageAPI{disk1, disk2}
@@ -327,10 +327,7 @@ func TestListDir(t *testing.T) {
 	}
 
 	// Should list "file1" from fsDir1.
-	entries, _, err := listDir(volume, "", "")
-	if err != nil {
-		t.Error(err)
-	}
+	entries, _ := listDir(volume, "", "")
 	if len(entries) != 2 {
 		t.Fatal("Expected the number of entries to be 2")
 	}
@@ -348,10 +345,7 @@ func TestListDir(t *testing.T) {
 	}
 
 	// Should list "file2" from fsDir2.
-	entries, _, err = listDir(volume, "", "")
-	if err != nil {
-		t.Error(err)
-	}
+	entries, _ = listDir(volume, "", "")
 	if len(entries) != 1 {
 		t.Fatal("Expected the number of entries to be 1")
 	}
@@ -362,13 +356,6 @@ func TestListDir(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	// None of the disks are available, should get
-	// errDiskNotFound. Since errDiskNotFound is an ignored error,
-	// we should get nil.
-	_, _, err = listDir(volume, "", "")
-	if err != nil {
-		t.Errorf("expected nil error but found %v.", err)
-	}
 }
 
 // TestRecursiveWalk - tests if treeWalk returns entries correctly with and
@@ -400,7 +387,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
 	}
 
 	// Create listDir function.
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1)
+	listDir := listDirFactory(context.Background(), isLeaf, disk1)
 
 	// Create the namespace.
 	var files = []string{
@@ -515,7 +502,7 @@ func TestSortedness(t *testing.T) {
 	}
 
 	// Create listDir function.
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1)
+	listDir := listDirFactory(context.Background(), isLeaf, disk1)
 
 	// Create the namespace.
 	var files = []string{
@@ -598,7 +585,7 @@ func TestTreeWalkIsEnd(t *testing.T) {
 	}
 
 	// Create listDir function.
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1)
+	listDir := listDirFactory(context.Background(), isLeaf, disk1)
 
 	// Create the namespace.
 	var files = []string{

cmd/ui-errors.go

@@ -19,8 +19,8 @@ package cmd
 var (
 	uiErrInvalidConfig = newUIErrFn(
 		"Invalid value found in the configuration file",
-		"Please ensure a valid value in the configuration file, for more details refer https://docs.minio.io/docs/minio-server-configuration-guide",
-		"",
+		"Please ensure a valid value in the configuration file",
+		"For more details, refer to https://docs.minio.io/docs/minio-server-configuration-guide",
 	)
 
 	uiErrInvalidBrowserValue = newUIErrFn(
@@ -120,12 +120,6 @@ Example 1:
 		"",
 	)
 
-	uiErrUnableToReadFromBackend = newUIErrFn(
-		"Unable to read from the backend",
-		"Please ensure Minio binary has read permissions for the backend",
-		"",
-	)
-
 	uiErrPortAlreadyInUse = newUIErrFn(
 		"Port is already in use",
 		"Please ensure no other program uses the same address/port",

cmd/utils.go

@@ -331,7 +331,7 @@ func ceilFrac(numerator, denominator int64) (ceil int64) {
 }
 
 // Returns context with ReqInfo details set in the context.
-func newContext(r *http.Request, api string) context.Context {
+func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -341,6 +341,7 @@ func newContext(r *http.Request, api string) context.Context {
 		object = prefix
 	}
 	reqInfo := &logger.ReqInfo{
+		RequestID:  w.Header().Get(responseRequestIDKey),
 		RemoteHost: handlers.GetSourceIP(r),
 		UserAgent:  r.Header.Get("user-agent"),
 		API:        api,

cmd/utils_test.go

@@ -445,6 +445,7 @@ func TestCeilFrac(t *testing.T) {
 // Test if isErrIgnored works correctly.
 func TestIsErrIgnored(t *testing.T) {
 	var errIgnored = fmt.Errorf("ignored error")
+	ignoredErrs := append(baseIgnoredErrs, errIgnored)
 	var testCases = []struct {
 		err     error
 		ignored bool
@@ -457,9 +458,13 @@ func TestIsErrIgnored(t *testing.T) {
 			err:     errIgnored,
 			ignored: true,
 		},
+		{
+			err:     errFaultyDisk,
+			ignored: true,
+		},
 	}
 	for i, testCase := range testCases {
-		if ok := IsErrIgnored(testCase.err, errIgnored); ok != testCase.ignored {
+		if ok := IsErrIgnored(testCase.err, ignoredErrs...); ok != testCase.ignored {
 			t.Errorf("Test: %d, Expected %t, got %t", i+1, testCase.ignored, ok)
 		}
 	}

cmd/xl-sets.go

@@ -622,8 +622,8 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir(). Sets passes set of disks.
-func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, treeWalkIgnoredErrs []error, sets ...[]StorageAPI) listDirFunc {
-	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string, err error) {
+func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, sets ...[]StorageAPI) listDirFunc {
+	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string) {
 		for _, disk := range disks {
 			if disk == nil {
 				continue
@@ -631,15 +631,10 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 
 			var entries []string
 			var newEntries []string
+			var err error
 			entries, err = disk.ListDir(bucket, prefixDir, -1)
 			if err != nil {
-				// For any reason disk was deleted or goes offline, continue
-				// and list from other disks if possible.
-				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
-					continue
-				}
-				logger.LogIf(ctx, err)
-				return nil, err
+				continue
 			}
 
 			// Find elements in entries which are not in mergedEntries
@@ -658,18 +653,13 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 				sort.Strings(mergedEntries)
 			}
 		}
-		return mergedEntries, nil
+		return mergedEntries
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
 		for _, disks := range sets {
-			var entries []string
-			entries, err = listDirInternal(bucket, prefixDir, prefixEntry, disks)
-			if err != nil {
-				return nil, false, err
-			}
-
+			entries := listDirInternal(bucket, prefixDir, prefixEntry, disks)
 			var newEntries []string
 			// Find elements in entries which are not in mergedEntries
 			for _, entry := range entries {
@@ -688,7 +678,7 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 			}
 		}
 		mergedEntries, delayIsLeaf = filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
-		return mergedEntries, delayIsLeaf, nil
+		return mergedEntries, delayIsLeaf
 	}
 	return listDir
 }
@@ -731,7 +721,7 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
 			setDisks = append(setDisks, set.getLoadBalancedDisks())
 		}
 
-		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, xlTreeWalkIgnoredErrs, setDisks...)
+		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, setDisks...)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 
@@ -1266,13 +1256,15 @@ func (s *xlSets) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir(). Sets passes set of disks.
 func listDirSetsHealFactory(isLeaf isLeafFunc, sets ...[]StorageAPI) listDirFunc {
-	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string, err error) {
+	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string) {
 		for _, disk := range disks {
 			if disk == nil {
 				continue
 			}
+
 			var entries []string
 			var newEntries []string
+			var err error
 			entries, err = disk.ListDir(bucket, prefixDir, -1)
 			if err != nil {
 				continue
@@ -1305,18 +1297,14 @@ func listDirSetsHealFactory(isLeaf isLeafFunc, sets ...[]StorageAPI) listDirFunc
 				sort.Strings(mergedEntries)
 			}
 		}
-		return mergedEntries, nil
+		return mergedEntries
 
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
 		for _, disks := range sets {
-			var entries []string
-			entries, err = listDirInternal(bucket, prefixDir, prefixEntry, disks)
-			if err != nil {
-				return nil, false, err
-			}
+			entries := listDirInternal(bucket, prefixDir, prefixEntry, disks)
 
 			var newEntries []string
 			// Find elements in entries which are not in mergedEntries
@@ -1335,7 +1323,7 @@ func listDirSetsHealFactory(isLeaf isLeafFunc, sets ...[]StorageAPI) listDirFunc
 				sort.Strings(mergedEntries)
 			}
 		}
-		return mergedEntries, false, nil
+		return mergedEntries, false
 	}
 	return listDir
 }

cmd/xl-v1-common.go

@@ -51,6 +51,8 @@ func (xl xlObjects) parentDirIsObject(ctx context.Context, bucket, parent string
 	return isParentDirObject(parent)
 }
 
+var xlTreeWalkIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errVolumeNotFound, errFileNotFound)
+
 // isObject - returns `true` if the prefix is an object i.e if
 // `xl.json` exists at the leaf, false otherwise.
 func (xl xlObjects) isObject(bucket, prefix string) (ok bool) {

cmd/xl-v1-healing-common.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
@@ -175,7 +176,10 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			// buffer is passed
 			_, hErr := onlineDisk.ReadFile(bucket, partPath, 0, buffer, verifier)
 
-			_, isCorrupt := hErr.(hashMismatchError)
+			isCorrupt := false
+			if hErr != nil {
+				isCorrupt = strings.HasPrefix(hErr.Error(), "Bitrot verification mismatch - expected ")
+			}
 			switch {
 			case isCorrupt:
 				fallthrough

cmd/xl-v1-list-objects.go

@@ -24,23 +24,19 @@ import (
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir()
-func listDirFactory(ctx context.Context, isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks ...StorageAPI) listDirFunc {
+func listDirFactory(ctx context.Context, isLeaf isLeafFunc, disks ...StorageAPI) listDirFunc {
 	// Returns sorted merged entries from all the disks.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
 		for _, disk := range disks {
 			if disk == nil {
 				continue
 			}
 			var entries []string
 			var newEntries []string
+			var err error
 			entries, err = disk.ListDir(bucket, prefixDir, -1)
 			if err != nil {
-				// For any reason disk was deleted or goes offline, continue
-				// and list from other disks if possible.
-				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
-					continue
-				}
-				return nil, false, err
+				continue
 			}
 
 			// Find elements in entries which are not in mergedEntries
@@ -60,7 +56,7 @@ func listDirFactory(ctx context.Context, isLeaf isLeafFunc, treeWalkIgnoredErrs
 			}
 		}
 		mergedEntries, delayIsLeaf = filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
-		return mergedEntries, delayIsLeaf, nil
+		return mergedEntries, delayIsLeaf
 	}
 	return listDir
 }
@@ -79,7 +75,7 @@ func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, del
 		endWalkCh = make(chan struct{})
 		isLeaf := xl.isObject
 		isLeafDir := xl.isObjectDir
-		listDir := listDirFactory(ctx, isLeaf, xlTreeWalkIgnoredErrs, xl.getLoadBalancedDisks()...)
+		listDir := listDirFactory(ctx, isLeaf, xl.getLoadBalancedDisks()...)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 

cmd/xl-v1.go

@@ -49,9 +49,6 @@ type xlObjects struct {
 	listPool *treeWalkPool
 }
 
-// list of all errors that can be ignored in tree walk operation in XL
-var xlTreeWalkIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errVolumeNotFound, errFileNotFound)
-
 // Shutdown function for object storage interface.
 func (xl xlObjects) Shutdown(ctx context.Context) error {
 	// Add any object layer shutdown activities here.

docs/distributed/README.md

@@ -48,8 +48,8 @@ To start a distributed Minio instance, you just need to pass drive locations as
 - Servers running distributed Minio instances should be less than 3 seconds apart. You can use [NTP](http://www.ntp.org/) as a best practice to ensure consistent times across servers. 
 - Running Distributed Minio on Windows is experimental as of now. Please proceed with caution. 
 
-Example 1: Start distributed Minio instance with 1 drive each on 8 nodes, by running this command on all the 8 nodes.
-
+Example 1: Start distributed Minio instance on 8 nodes with 1 drive each (pictured below), by running this command on all the 8 nodes:
+![Distributed Minio, 8 nodes with 1 disk each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_8.jpg?raw=true)
 #### GNU/Linux and macOS
 
 ```shell
@@ -60,6 +60,7 @@ minio server http://192.168.1.11/export1 http://192.168.1.12/export2 \
                http://192.168.1.15/export5 http://192.168.1.16/export6 \
                http://192.168.1.17/export7 http://192.168.1.18/export8
 ```
+
 #### Windows 
 
 ```cmd
@@ -71,23 +72,30 @@ minio.exe server http://192.168.1.11/C:/data http://192.168.1.12/C:/data ^
                   http://192.168.1.17/C:/data http://192.168.1.18/C:/data
 ```
 
-![Distributed Minio, 8 nodes with 1 disk each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_8.jpg?raw=true)
 
-Example 2: Start distributed Minio instance with 4 drives each on 4 nodes, by running this command on all the 4 nodes.
+Example 2: Start distributed Minio instance on 4 nodes with 4 drives (pictured below), by running this command on all the 4 nodes:
+![Distributed Minio, 4 nodes with 4 disks each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_16.jpg?raw=true)
 
 #### GNU/Linux and macOS
 
 ```shell
 export MINIO_ACCESS_KEY=<ACCESS_KEY>
 export MINIO_SECRET_KEY=<SECRET_KEY>
-minio server http://192.168.1.11/export1 http://192.168.1.11/export2 \
-               http://192.168.1.11/export3 http://192.168.1.11/export4 \
-               http://192.168.1.12/export1 http://192.168.1.12/export2 \
-               http://192.168.1.12/export3 http://192.168.1.12/export4 \
-               http://192.168.1.13/export1 http://192.168.1.13/export2 \
-               http://192.168.1.13/export3 http://192.168.1.13/export4 \
-               http://192.168.1.14/export1 http://192.168.1.14/export2 \
-               http://192.168.1.14/export3 http://192.168.1.14/export4
+minio server http://192.168.1.11/export1 http://192.168.1.12/export1 \
+               http://192.168.1.13/export1 http://192.168.1.14/export1 \
+               http://192.168.1.11/export2 http://192.168.1.12/export2 \
+               http://192.168.1.13/export2 http://192.168.1.14/export2 \
+               http://192.168.1.11/export3 http://192.168.1.12/export3 \
+               http://192.168.1.13/export3 http://192.168.1.14/export3 \
+               http://192.168.1.11/export4 http://192.168.1.12/export4 \
+               http://192.168.1.13/export4 http://192.168.1.14/export4
+```
+
+##### Short version of the same command:
+```shell
+export MINIO_ACCESS_KEY=<ACCESS_KEY>
+export MINIO_SECRET_KEY=<SECRET_KEY>
+minio server http://192.168.1.{11...14}/export{1...4}
 ```
 
 #### Windows
@@ -95,17 +103,24 @@ minio server http://192.168.1.11/export1 http://192.168.1.11/export2 \
 ```cmd
 set MINIO_ACCESS_KEY=<ACCESS_KEY>
 set MINIO_SECRET_KEY=<SECRET_KEY>
-minio.exe server http://192.168.1.11/C:/data1 http://192.168.1.11/C:/data2 ^
-                  http://192.168.1.11/C:/data3 http://192.168.1.11/C:/data4 ^
-                  http://192.168.1.12/C:/data1 http://192.168.1.12/C:/data2 ^
-                  http://192.168.1.12/C:/data3 http://192.168.1.12/C:/data4 ^
-                  http://192.168.1.13/C:/data1 http://192.168.1.13/C:/data2 ^
-                  http://192.168.1.13/C:/data3 http://192.168.1.13/C:/data4 ^                  
-                  http://192.168.1.14/C:/data1 http://192.168.1.14/C:/data2 ^
-                  http://192.168.1.14/C:/data3 http://192.168.1.14/C:/data4
+minio.exe server http://192.168.1.11/C:/data1 http://192.168.1.12/C:/data1 ^
+                  http://192.168.1.13/C:/data1 http://192.168.1.14/C:/data1 ^
+                  http://192.168.1.11/C:/data2 http://192.168.1.12/C:/data2 ^
+                  http://192.168.1.13/C:/data2 http://192.168.1.14/C:/data2 ^
+                  http://192.168.1.11/C:/data3 http://192.168.1.12/C:/data3 ^
+                  http://192.168.1.13/C:/data3 http://192.168.1.14/C:/data3 ^
+                  http://192.168.1.11/C:/data4 http://192.168.1.12/C:/data4 ^
+                  http://192.168.1.13/C:/data4 http://192.168.1.14/C:/data4
 ```
 
-![Distributed Minio, 4 nodes with 4 disks each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_16.jpg?raw=true)
+##### Short version of the same command:
+```cmd
+set MINIO_ACCESS_KEY=<ACCESS_KEY>
+set MINIO_SECRET_KEY=<SECRET_KEY>
+minio.exe server http://192.168.1.{1...14}/C:/data{1...4}
+```
+__NOTE:__ `{1...n}` shown in shortened examples above have 3 dots! Using only 2 dots `{1..4}` will be interpreted by your shell and won't be passed to minio server, affecting the erasure coding order, which may impact performance and high availability. __Always use `{1...n}` (3 dots!) to allow minio server to optimally stripe erasure-coded data__
+
 
 ## 3. Test your setup
 

docs/gateway/s3.md

@@ -0,0 +1,72 @@
+# Minio S3 Gateway [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+
+Minio S3 Gateway adds Minio features like Minio Browser and disk caching to AWS S3 or any other AWS S3 compatible service.
+
+## Run Minio Gateway for AWS S3
+
+As a prerequisite to run Minio S3 gateway, you need valid AWS S3 access key and secret key.
+
+### Using Docker
+
+```
+docker run -p 9000:9000 --name minio-s3 \
+ -e "MINIO_ACCESS_KEY=aws_s3_access_key" \
+ -e "MINIO_SECRET_KEY=aws_s3_secret_key" \
+ minio/minio gateway s3
+```
+
+### Using Binary
+
+```
+export MINIO_ACCESS_KEY=aws_s3_access_key
+export MINIO_SECRET_KEY=aws_s3_secret_key
+minio gateway s3
+```
+
+## Run Minio Gateway for AWS S3 compatible services
+
+As a prerequisite to run Minio S3 gateway on an AWS S3 compatible service, you need valid access key, secret key and service endpoint.
+
+### Using Docker
+
+```
+docker run -p 9000:9000 --name minio-s3 \
+ -e "MINIO_ACCESS_KEY=access_key" \
+ -e "MINIO_SECRET_KEY=secret_key" \
+ minio/minio gateway s3 --address https://s3_compatible_service_endpoint:port
+```
+
+### Using Binary
+
+```
+export MINIO_ACCESS_KEY=access_key
+export MINIO_SECRET_KEY=secret_key
+minio gateway s3 --address https://s3_compatible_service_endpoint:port
+```
+
+## Minio Caching
+
+Minio edge caching allows storing content closer to the applications. Frequently accessed objects are stored in a local disk based cache. Edge caching with Minio gateway feature allows
+
+- Dramatic improvements for time to first byte for any object.
+- Avoid S3 [data transfer charges](https://aws.amazon.com/s3/pricing/).
+
+Refer [this document](https://docs.minio.io/docs/minio-disk-cache-guide.html) to get started with Minio Caching.
+
+## Minio Browser
+
+Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully.
+
+![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser-gateway.png?raw=true)
+
+With Minio S3 gateway, you can use Minio browser to explore AWS S3 based objects.
+
+### Known limitations
+
+- Bucket notification APIs are not supported.
+
+## Explore Further
+
+- [`mc` command-line interface](https://docs.minio.io/docs/minio-client-quickstart-guide)
+- [`aws` command-line interface](https://docs.minio.io/docs/aws-cli-with-minio)
+- [`minio-go` Go SDK](https://docs.minio.io/docs/golang-client-quickstart-guide)

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
  minio1:
-  image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+  image: minio/minio:RELEASE.2018-07-23T18-34-49Z
   volumes:
    - data1:/data
   ports:
@@ -15,7 +15,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio2:
-  image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+  image: minio/minio:RELEASE.2018-07-23T18-34-49Z
   volumes:
    - data2:/data
   ports:
@@ -25,7 +25,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio3:
-  image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+  image: minio/minio:RELEASE.2018-07-23T18-34-49Z
   volumes:
    - data3:/data
   ports:
@@ -35,7 +35,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio4:
-  image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+  image: minio/minio:RELEASE.2018-07-23T18-34-49Z
   volumes:
    - data4:/data
   ports:

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+    image: minio/minio:RELEASE.2018-07-23T18-34-49Z
     volumes:
       - minio1-data:/export
     ports:
@@ -20,7 +20,7 @@ services:
       - access_key
 
   minio2:
-    image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+    image: minio/minio:RELEASE.2018-07-23T18-34-49Z
     volumes:
       - minio2-data:/export
     ports:
@@ -38,7 +38,7 @@ services:
       - access_key
 
   minio3:
-    image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+    image: minio/minio:RELEASE.2018-07-23T18-34-49Z
     volumes:
       - minio3-data:/export
     ports:
@@ -56,7 +56,7 @@ services:
       - access_key
 
   minio4:
-    image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+    image: minio/minio:RELEASE.2018-07-23T18-34-49Z
     volumes:
       - minio4-data:/export
     ports:

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+    image: minio/minio:RELEASE.2018-07-23T18-34-49Z
     volumes:
       - minio1-data:/export
     ports:
@@ -20,7 +20,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio2:
-    image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+    image: minio/minio:RELEASE.2018-07-23T18-34-49Z
     volumes:
       - minio2-data:/export
     ports:
@@ -38,7 +38,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio3:
-    image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+    image: minio/minio:RELEASE.2018-07-23T18-34-49Z
     volumes:
       - minio3-data:/export
     ports:
@@ -56,7 +56,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio4:
-    image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+    image: minio/minio:RELEASE.2018-07-23T18-34-49Z
     volumes:
       - minio4-data:/export
     ports:

docs/orchestration/kubernetes-yaml/README.md

@@ -124,7 +124,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+        image: minio/minio:RELEASE.2018-07-23T18-34-49Z
         args:
         - server
         - /data
@@ -303,7 +303,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+        image: minio/minio:RELEASE.2018-07-23T18-34-49Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data
@@ -514,7 +514,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+        image: minio/minio:RELEASE.2018-07-23T18-34-49Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes-yaml/minio-distributed-statefulset.yaml

@@ -21,7 +21,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+        image: minio/minio:RELEASE.2018-07-23T18-34-49Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes-yaml/minio-gcs-gateway-deployment.yaml

@@ -21,7 +21,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+        image: minio/minio:RELEASE.2018-07-23T18-34-49Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes-yaml/minio-standalone-deployment.yaml

@@ -29,7 +29,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+        image: minio/minio:RELEASE.2018-07-23T18-34-49Z
         args:
         - server
         - /data

docs/orchestration/kubernetes/README.md

@@ -30,7 +30,7 @@ Above command deploys Minio on the Kubernetes cluster in the default configurati
 | Parameter                  | Description                         | Default                                                 |
 |----------------------------|-------------------------------------|---------------------------------------------------------|
 | `image`                    | Minio image name                    | `minio/minio`                                           |
-| `imageTag`                 | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-07-10T01-42-11Z`|
+| `imageTag`                 | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-07-23T18-34-49Z`|
 | `imagePullPolicy`          | Image pull policy                   | `Always`                                                |
 | `mode`                     | Minio server mode (`standalone`, `shared` or `distributed`)| `standalone`                     |
 | `numberOfNodes`            | Number of nodes (applicable only for Minio distributed mode). Should be 4 <= x <= 16 | `4`    |

docs/orchestration/minikube/statefulset.yaml

@@ -54,7 +54,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-07-10T01-42-11Z
+        image: minio/minio:RELEASE.2018-07-23T18-34-49Z
         imagePullPolicy: IfNotPresent
         args: ["server", "http://minio-0.minio.default.svc.cluster.local/data", "http://minio-1.minio.default.svc.cluster.local/data", "http://minio-2.minio.default.svc.cluster.local/data", "http://minio-3.minio.default.svc.cluster.local/data"]
         ports:

pkg/event/target/amqp.go

@@ -43,6 +43,17 @@ type AMQPArgs struct {
 	AutoDeleted  bool     `json:"autoDeleted"`
 }
 
+// Validate AMQP arguments
+func (a *AMQPArgs) Validate() error {
+	if !a.Enable {
+		return nil
+	}
+	if _, err := amqp.ParseURI(a.URL.String()); err != nil {
+		return err
+	}
+	return nil
+}
+
 // AMQPTarget - AMQP target
 type AMQPTarget struct {
 	id        event.TargetID

pkg/event/target/elasticsearch.go

@@ -18,8 +18,10 @@ package target
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/pkg/event"
@@ -36,6 +38,26 @@ type ElasticsearchArgs struct {
 	Index  string   `json:"index"`
 }
 
+// Validate ElasticsearchArgs fields
+func (a ElasticsearchArgs) Validate() error {
+	if !a.Enable {
+		return nil
+	}
+	if a.URL.IsEmpty() {
+		return errors.New("empty URL")
+	}
+	if a.Format != "" {
+		f := strings.ToLower(a.Format)
+		if f != event.NamespaceFormat && f != event.AccessFormat {
+			return errors.New("format value unrecognized")
+		}
+	}
+	if a.Index == "" {
+		return errors.New("empty index value")
+	}
+	return nil
+}
+
 // ElasticsearchTarget - Elasticsearch target.
 type ElasticsearchTarget struct {
 	id     event.TargetID

pkg/event/target/kafka.go

@@ -18,6 +18,7 @@ package target
 
 import (
 	"encoding/json"
+	"errors"
 	"net/url"
 
 	"github.com/minio/minio/pkg/event"
@@ -33,6 +34,22 @@ type KafkaArgs struct {
 	Topic   string      `json:"topic"`
 }
 
+// Validate KafkaArgs fields
+func (k KafkaArgs) Validate() error {
+	if !k.Enable {
+		return nil
+	}
+	if len(k.Brokers) == 0 {
+		return errors.New("no broker address found")
+	}
+	for _, b := range k.Brokers {
+		if _, err := xnet.ParseHost(b.String()); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // KafkaTarget - Kafka target.
 type KafkaTarget struct {
 	id       event.TargetID

pkg/event/target/mqtt.go

@@ -20,6 +20,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"net/url"
 	"time"
 
@@ -42,6 +43,23 @@ type MQTTArgs struct {
 	RootCAs              *x509.CertPool `json:"-"`
 }
 
+// Validate MQTTArgs fields
+func (m MQTTArgs) Validate() error {
+	if !m.Enable {
+		return nil
+	}
+	u, err := xnet.ParseURL(m.Broker.String())
+	if err != nil {
+		return err
+	}
+	switch u.Scheme {
+	case "ws", "wss", "tcp", "ssl", "tls", "tcps":
+	default:
+		return errors.New("unknown protocol in broker address")
+	}
+	return nil
+}
+
 // MQTTTarget - MQTT target.
 type MQTTTarget struct {
 	id     event.TargetID

pkg/event/target/mysql.go

@@ -58,6 +58,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/go-sql-driver/mysql"
@@ -88,6 +90,42 @@ type MySQLArgs struct {
 	Database string   `json:"database"`
 }
 
+// Validate MySQLArgs fields
+func (m MySQLArgs) Validate() error {
+	if !m.Enable {
+		return nil
+	}
+
+	if m.Format != "" {
+		f := strings.ToLower(m.Format)
+		if f != event.NamespaceFormat && f != event.AccessFormat {
+			return fmt.Errorf("unrecognized format")
+		}
+	}
+
+	if m.Table == "" {
+		return fmt.Errorf("table unspecified")
+	}
+
+	if m.DSN != "" {
+		if _, err := mysql.ParseDSN(m.DSN); err != nil {
+			return err
+		}
+	} else {
+		// Some fields need to be specified when DSN is unspecified
+		if m.Port == "" {
+			return fmt.Errorf("unspecified port")
+		}
+		if _, err := strconv.Atoi(m.Port); err != nil {
+			return fmt.Errorf("invalid port")
+		}
+		if m.Database == "" {
+			return fmt.Errorf("database unspecified")
+		}
+	}
+	return nil
+}
+
 // MySQLTarget - MySQL target.
 type MySQLTarget struct {
 	id         event.TargetID

pkg/event/target/nats.go

@@ -18,6 +18,7 @@ package target
 
 import (
 	"encoding/json"
+	"errors"
 	"net/url"
 
 	"github.com/minio/minio/pkg/event"
@@ -45,6 +46,32 @@ type NATSArgs struct {
 	} `json:"streaming"`
 }
 
+// Validate NATSArgs fields
+func (n NATSArgs) Validate() error {
+	if !n.Enable {
+		return nil
+	}
+
+	if n.Address.IsEmpty() {
+		return errors.New("empty address")
+	}
+
+	if n.Subject == "" {
+		return errors.New("empty subject")
+	}
+
+	if n.Streaming.Enable {
+		if n.Streaming.ClusterID == "" {
+			return errors.New("empty cluster id")
+		}
+		if n.Streaming.ClientID == "" {
+			return errors.New("empty client id")
+		}
+	}
+
+	return nil
+}
+
 // NATSTarget - NATS target.
 type NATSTarget struct {
 	id       event.TargetID

pkg/event/target/postgresql.go

@@ -58,10 +58,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
-	_ "github.com/lib/pq" // Register postgres driver
+	"github.com/lib/pq" // Register postgres driver
 	"github.com/minio/minio/pkg/event"
 	xnet "github.com/minio/minio/pkg/net"
 )
@@ -89,6 +90,41 @@ type PostgreSQLArgs struct {
 	Database         string   `json:"database"` // default: same as user
 }
 
+// Validate PostgreSQLArgs fields
+func (p PostgreSQLArgs) Validate() error {
+	if !p.Enable {
+		return nil
+	}
+	if p.Table == "" {
+		return fmt.Errorf("empty table name")
+	}
+	if p.Format != "" {
+		f := strings.ToLower(p.Format)
+		if f != event.NamespaceFormat && f != event.AccessFormat {
+			return fmt.Errorf("unrecognized format value")
+		}
+	}
+
+	if p.ConnectionString != "" {
+		if _, err := pq.ParseURL(p.ConnectionString); err != nil {
+			return err
+		}
+	} else {
+		// Some fields need to be specified when ConnectionString is unspecified
+		if p.Port == "" {
+			return fmt.Errorf("unspecified port")
+		}
+		if _, err := strconv.Atoi(p.Port); err != nil {
+			return fmt.Errorf("invalid port")
+		}
+		if p.Database == "" {
+			return fmt.Errorf("database unspecified")
+		}
+	}
+
+	return nil
+}
+
 // PostgreSQLTarget - PostgreSQL target.
 type PostgreSQLTarget struct {
 	id         event.TargetID

pkg/event/target/redis.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/garyburd/redigo/redis"
@@ -36,6 +37,26 @@ type RedisArgs struct {
 	Key      string    `json:"key"`
 }
 
+// Validate RedisArgs fields
+func (r RedisArgs) Validate() error {
+	if !r.Enable {
+		return nil
+	}
+
+	if r.Format != "" {
+		f := strings.ToLower(r.Format)
+		if f != event.NamespaceFormat && f != event.AccessFormat {
+			return fmt.Errorf("unrecognized format")
+		}
+	}
+
+	if r.Key == "" {
+		return fmt.Errorf("empty key")
+	}
+
+	return nil
+}
+
 // RedisTarget - Redis target.
 type RedisTarget struct {
 	id   event.TargetID

pkg/event/target/webhook.go

@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -38,6 +39,17 @@ type WebhookArgs struct {
 	RootCAs  *x509.CertPool `json:"-"`
 }
 
+// Validate WebhookArgs fields
+func (w WebhookArgs) Validate() error {
+	if !w.Enable {
+		return nil
+	}
+	if w.Endpoint.IsEmpty() {
+		return errors.New("endpoint empty")
+	}
+	return nil
+}
+
 // WebhookTarget - Webhook target.
 type WebhookTarget struct {
 	id         event.TargetID

pkg/handlers/proxy.go

@@ -30,7 +30,6 @@ var (
 	xForwardedPort   = http.CanonicalHeaderKey("X-Forwarded-Port")
 	xForwardedProto  = http.CanonicalHeaderKey("X-Forwarded-Proto")
 	xForwardedScheme = http.CanonicalHeaderKey("X-Forwarded-Scheme")
-	xForwardedServer = http.CanonicalHeaderKey("X-Forwarded-Server")
 	xRealIP          = http.CanonicalHeaderKey("X-Real-IP")
 )
 

pkg/lock/lock_solaris.go

@@ -98,3 +98,8 @@ func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, er
 func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
 	return lockedOpenFile(path, flag, perm, syscall.F_SETLKW)
 }
+
+// Open - Call os.OpenFile
+func Open(path string, flag int, perm os.FileMode) (*os.File, error) {
+	return os.OpenFile(path, flag, perm)
+}

pkg/madmin/utils.go

@@ -17,7 +17,6 @@
 package madmin
 
 import (
-	"crypto/md5"
 	"encoding/json"
 	"io"
 	"io/ioutil"
@@ -38,13 +37,6 @@ func sum256(data []byte) []byte {
 	return hash.Sum(nil)
 }
 
-// sumMD5 calculate sumMD5 sum for an input byte array.
-func sumMD5(data []byte) []byte {
-	hash := md5.New()
-	hash.Write(data)
-	return hash.Sum(nil)
-}
-
 // jsonDecoder decode json to go type.
 func jsonDecoder(body io.Reader, v interface{}) error {
 	d := json.NewDecoder(body)