Fix admin API doc formatting (#6235)

No locks are ever left in memory, we also
have a periodic interval of clearing stale locks
anyways. The lock instrumentation was not complete
and was seldom used.

Deprecate this for now and bring it back later if
it is really needed. This also in-turn seems to improve
performance slightly.

.travis.yml

@@ -22,21 +22,9 @@ matrix:
         - make coverage
         - node --version
         - cd browser && yarn && yarn test && cd ..
-    - os: linux-ppc64le
-      env:
-        - ARCH=ppc64le
-      go: 1.10.1
-      script:
-        - make
-        - diff -au <(gofmt -s -d cmd) <(printf "")
-        - diff -au <(gofmt -s -d pkg) <(printf "")
-        - make test GOFLAGS="-timeout 15m -v"
-        - make coverage
-        - node --version
-        - cd browser && yarn && yarn test && cd ..
 
 before_install:
 - nvm install stable
 
 after_success:
-- bash <(curl -s https://codecov.io/bash)
+- bash <(curl -s https://codecov.io/bash)

browser/app/less/inc/list.less

@@ -100,6 +100,12 @@ div.fesl-row {
         }
     }
 
+    .fesl-item-name {
+        a {
+            cursor: default;
+        }
+    }
+
     /*--------------------------
         Icons
     ----------------------------*/

cmd/acl-handlers.go

@@ -54,7 +54,7 @@ type accessControlPolicy struct {
 // This operation uses the ACL
 // subresource to return the ACL of a specified bucket.
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketACL")
+	ctx := newContext(r, w, "GetBucketACL")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -101,7 +101,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 // This operation uses the ACL
 // subresource to return the ACL of a specified object.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetObjectACL")
+	ctx := newContext(r, w, "GetObjectACL")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/admin-handlers.go

@@ -275,133 +275,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-// validateLockQueryParams - Validates query params for list/clear
-// locks management APIs.
-func validateLockQueryParams(vars url.Values) (string, string, time.Duration,
-	APIErrorCode) {
-
-	bucket := vars.Get(string(mgmtBucket))
-	prefix := vars.Get(string(mgmtPrefix))
-	olderThanStr := vars.Get(string(mgmtLockOlderThan))
-
-	// N B empty bucket name is invalid
-	if !IsValidBucketName(bucket) {
-		return "", "", time.Duration(0), ErrInvalidBucketName
-	}
-	// empty prefix is valid.
-	if !IsValidObjectPrefix(prefix) {
-		return "", "", time.Duration(0), ErrInvalidObjectName
-	}
-
-	// If older-than parameter was empty then set it to 0s to list
-	// all locks older than now.
-	if olderThanStr == "" {
-		olderThanStr = "0s"
-	}
-	duration, err := time.ParseDuration(olderThanStr)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return "", "", time.Duration(0), ErrInvalidDuration
-	}
-
-	return bucket, prefix, duration, ErrNone
-}
-
-// ListLocksHandler - GET /minio/admin/v1/locks?bucket=mybucket&prefix=myprefix&older-than=10s
-// - bucket is a mandatory query parameter
-// - prefix and older-than are optional query parameters
-// ---------
-// Lists locks held on a given bucket, prefix and duration it was held for.
-func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Request) {
-
-	adminAPIErr := checkAdminRequestAuthType(r, "")
-	if adminAPIErr != ErrNone {
-		writeErrorResponseJSON(w, adminAPIErr, r.URL)
-		return
-	}
-
-	vars := r.URL.Query()
-	bucket, prefix, duration, adminAPIErr := validateLockQueryParams(vars)
-	if adminAPIErr != ErrNone {
-		writeErrorResponseJSON(w, adminAPIErr, r.URL)
-		return
-	}
-
-	// Fetch lock information of locks matching bucket/prefix that
-	// are available for longer than duration.
-	volLocks, err := listPeerLocksInfo(globalAdminPeers, bucket, prefix,
-		duration)
-	if err != nil {
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		logger.LogIf(context.Background(), err)
-		return
-	}
-
-	// Marshal list of locks as json.
-	jsonBytes, err := json.Marshal(volLocks)
-	if err != nil {
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		logger.LogIf(context.Background(), err)
-		return
-	}
-
-	// Reply with list of locks held on bucket, matching prefix
-	// held longer than duration supplied, as json.
-	writeSuccessResponseJSON(w, jsonBytes)
-}
-
-// ClearLocksHandler - DELETE /minio/admin/v1/locks?bucket=mybucket&prefix=myprefix&duration=duration
-// - bucket is a mandatory query parameter
-// - prefix and older-than are optional query parameters
-// ---------
-// Clear locks held on a given bucket, prefix and duration it was held for.
-func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ClearLocks")
-
-	// Get object layer instance.
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-
-	adminAPIErr := checkAdminRequestAuthType(r, "")
-	if adminAPIErr != ErrNone {
-		writeErrorResponseJSON(w, adminAPIErr, r.URL)
-		return
-	}
-
-	vars := r.URL.Query()
-	bucket, prefix, duration, adminAPIErr := validateLockQueryParams(vars)
-	if adminAPIErr != ErrNone {
-		writeErrorResponseJSON(w, adminAPIErr, r.URL)
-		return
-	}
-
-	// Fetch lock information of locks matching bucket/prefix that
-	// are held for longer than duration.
-	volLocks, err := listPeerLocksInfo(globalAdminPeers, bucket, prefix,
-		duration)
-	if err != nil {
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		logger.LogIf(ctx, err)
-		return
-	}
-
-	// Marshal list of locks as json.
-	jsonBytes, err := json.Marshal(volLocks)
-	if err != nil {
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		logger.LogIf(ctx, err)
-		return
-	}
-
-	objLayer.ClearLocks(ctx, volLocks)
-
-	// Reply with list of locks cleared, as json.
-	writeSuccessResponseJSON(w, jsonBytes)
-}
-
 // extractHealInitParams - Validates params for heal init API.
 func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 	hs madmin.HealOpts, clientToken string, forceStart bool,
@@ -464,7 +337,7 @@ func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 // sequence. However, if the force-start flag is provided, the server
 // aborts the running heal sequence and starts a new one.
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "Heal")
+	ctx := newContext(r, w, "Heal")
 
 	// Get object layer instance.
 	objLayer := newObjectLayerFn()
@@ -712,7 +585,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	err = json.Unmarshal(configBytes, &config)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
+		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
 		return
 	}
 
@@ -727,6 +600,11 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		}
 	}
 
+	if err := config.Validate(); err != nil {
+		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
+		return
+	}
+
 	// Write config received from request onto a temporary file on
 	// all nodes.
 	tmpFileName := fmt.Sprintf(minioConfigTmpFormat, mustGetUUID())

cmd/admin-handlers_test.go

@@ -38,22 +38,24 @@ import (
 
 var (
 	configJSON = []byte(`{
-	"version": "13",
+	"version": "27",
 	"credential": {
 		"accessKey": "minio",
 		"secretKey": "minio123"
 	},
-	"region": "us-west-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "fatal"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
+	"region": "",
+	"browser": "on",
+	"worm": "off",
+	"domain": "",
+	"storageclass": {
+		"standard": "",
+		"rrs": ""
+	},
+	"cache": {
+		"drives": [],
+		"expiry": 90,
+		"maxuse": 80,
+		"exclude": []
 	},
 	"notify": {
 		"amqp": {
@@ -63,6 +65,7 @@ var (
 				"exchange": "",
 				"routingKey": "",
 				"exchangeType": "",
+				"deliveryMode": 0,
 				"mandatory": false,
 				"immediate": false,
 				"durable": false,
@@ -71,6 +74,47 @@ var (
 				"autoDeleted": false
 			}
 		},
+		"elasticsearch": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"url": "",
+				"index": ""
+			}
+		},
+		"kafka": {
+			"1": {
+				"enable": false,
+				"brokers": null,
+				"topic": ""
+			}
+		},
+		"mqtt": {
+			"1": {
+				"enable": false,
+				"broker": "",
+				"topic": "",
+				"qos": 0,
+				"clientId": "",
+				"username": "",
+				"password": "",
+				"reconnectInterval": 0,
+				"keepAliveInterval": 0
+			}
+		},
+		"mysql": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"dsnString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
 		"nats": {
 			"1": {
 				"enable": false,
@@ -90,24 +134,10 @@ var (
 				}
 			}
 		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
 		"postgresql": {
 			"1": {
 				"enable": false,
+				"format": "",
 				"connectionString": "",
 				"table": "",
 				"host": "",
@@ -117,11 +147,13 @@ var (
 				"database": ""
 			}
 		},
-		"kafka": {
+		"redis": {
 			"1": {
 				"enable": false,
-				"brokers": null,
-				"topic": ""
+				"format": "",
+				"address": "",
+				"password": "",
+				"key": ""
 			}
 		},
 		"webhook": {
@@ -130,8 +162,20 @@ var (
 				"endpoint": ""
 			}
 		}
-	}
-}`)
+	    },
+	    "logger": {
+		"console": {
+		    "enabled": true
+		},
+		"http": {
+		    "1": {
+			"enabled": false,
+			"endpoint": "http://user:example@localhost:9001/api/endpoint"
+		    }
+		}
+	    }
+
+	}`)
 )
 
 // adminXLTestBed - encapsulates subsystems that need to be setup for
@@ -176,10 +220,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// Init global heal state
 	initAllHealState(globalIsXL)
 
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
-	if err != nil {
-		return nil, err
-	}
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
@@ -570,193 +611,6 @@ func TestServiceSetCreds(t *testing.T) {
 	}
 }
 
-// mkLockQueryVal - helper function to build lock query param.
-func mkLockQueryVal(bucket, prefix, durationStr string) url.Values {
-	qVal := url.Values{}
-	qVal.Set(string(mgmtBucket), bucket)
-	qVal.Set(string(mgmtPrefix), prefix)
-	qVal.Set(string(mgmtLockOlderThan), durationStr)
-	return qVal
-}
-
-// Test for locks list management REST API.
-func TestListLocksHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// Initialize admin peers to make admin RPC calls.
-	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
-
-	testCases := []struct {
-		bucket         string
-		prefix         string
-		duration       string
-		expectedStatus int
-	}{
-		// Test 1 - valid testcase
-		{
-			bucket:         "mybucket",
-			prefix:         "myobject",
-			duration:       "1s",
-			expectedStatus: http.StatusOK,
-		},
-		// Test 2 - invalid duration
-		{
-			bucket:         "mybucket",
-			prefix:         "myprefix",
-			duration:       "invalidDuration",
-			expectedStatus: http.StatusBadRequest,
-		},
-		// Test 3 - invalid bucket name
-		{
-			bucket:         `invalid\\Bucket`,
-			prefix:         "myprefix",
-			duration:       "1h",
-			expectedStatus: http.StatusBadRequest,
-		},
-		// Test 4 - invalid prefix
-		{
-			bucket:         "mybucket",
-			prefix:         `invalid\\Prefix`,
-			duration:       "1h",
-			expectedStatus: http.StatusBadRequest,
-		},
-	}
-
-	for i, test := range testCases {
-		queryVal := mkLockQueryVal(test.bucket, test.prefix, test.duration)
-		req, err := newTestRequest("GET", "/minio/admin/v1/locks?"+queryVal.Encode(), 0, nil)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to construct list locks request - %v", i+1, err)
-		}
-
-		cred := globalServerConfig.GetCredential()
-		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
-		}
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if test.expectedStatus != rec.Code {
-			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
-		}
-	}
-}
-
-// Test for locks clear management REST API.
-func TestClearLocksHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// Initialize admin peers to make admin RPC calls.
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
-
-	testCases := []struct {
-		bucket         string
-		prefix         string
-		duration       string
-		expectedStatus int
-	}{
-		// Test 1 - valid testcase
-		{
-			bucket:         "mybucket",
-			prefix:         "myobject",
-			duration:       "1s",
-			expectedStatus: http.StatusOK,
-		},
-		// Test 2 - invalid duration
-		{
-			bucket:         "mybucket",
-			prefix:         "myprefix",
-			duration:       "invalidDuration",
-			expectedStatus: http.StatusBadRequest,
-		},
-		// Test 3 - invalid bucket name
-		{
-			bucket:         `invalid\\Bucket`,
-			prefix:         "myprefix",
-			duration:       "1h",
-			expectedStatus: http.StatusBadRequest,
-		},
-		// Test 4 - invalid prefix
-		{
-			bucket:         "mybucket",
-			prefix:         `invalid\\Prefix`,
-			duration:       "1h",
-			expectedStatus: http.StatusBadRequest,
-		},
-	}
-
-	for i, test := range testCases {
-		queryVal := mkLockQueryVal(test.bucket, test.prefix, test.duration)
-		req, err := newTestRequest("DELETE", "/minio/admin/v1/locks?"+queryVal.Encode(), 0, nil)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to construct clear locks request - %v", i+1, err)
-		}
-
-		cred := globalServerConfig.GetCredential()
-		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
-		}
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if test.expectedStatus != rec.Code {
-			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
-		}
-	}
-}
-
-// Test for lock query param validation helper function.
-func TestValidateLockQueryParams(t *testing.T) {
-	// reset globals.
-	// this is to make sure that the tests are not affected by modified globals.
-	resetTestGlobals()
-	// initialize NSLock.
-	initNSLock(false)
-	// Sample query values for test cases.
-	allValidVal := mkLockQueryVal("bucket", "prefix", "1s")
-	invalidBucketVal := mkLockQueryVal(`invalid\\Bucket`, "prefix", "1s")
-	invalidPrefixVal := mkLockQueryVal("bucket", `invalid\\Prefix`, "1s")
-	invalidOlderThanVal := mkLockQueryVal("bucket", "prefix", "invalidDuration")
-
-	testCases := []struct {
-		qVals  url.Values
-		apiErr APIErrorCode
-	}{
-		{
-			qVals:  invalidBucketVal,
-			apiErr: ErrInvalidBucketName,
-		},
-		{
-			qVals:  invalidPrefixVal,
-			apiErr: ErrInvalidObjectName,
-		},
-		{
-			qVals:  invalidOlderThanVal,
-			apiErr: ErrInvalidDuration,
-		},
-		{
-			qVals:  allValidVal,
-			apiErr: ErrNone,
-		},
-	}
-
-	for i, test := range testCases {
-		_, _, _, apiErr := validateLockQueryParams(test.qVals)
-		if apiErr != test.apiErr {
-			t.Errorf("Test %d - Expected error %v but received %v", i+1, test.apiErr, apiErr)
-		}
-	}
-}
-
 // buildAdminRequest - helper function to build an admin API request.
 func buildAdminRequest(queryVal url.Values, method, path string,
 	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {

cmd/admin-router.go

@@ -53,13 +53,6 @@ func registerAdminRouter(router *mux.Router) {
 	// Info operations
 	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 
-	/// Lock operations
-
-	// List Locks
-	adminV1Router.Methods(http.MethodGet).Path("/locks").HandlerFunc(httpTraceAll(adminAPI.ListLocksHandler))
-	// Clear locks
-	adminV1Router.Methods(http.MethodDelete).Path("/locks").HandlerFunc(httpTraceAll(adminAPI.ClearLocksHandler))
-
 	/// Heal operations
 
 	// Heal processing endpoint.

cmd/admin-rpc-client.go

@@ -54,19 +54,6 @@ func (rpcClient *AdminRPCClient) ReInitFormat(dryRun bool) error {
 	return rpcClient.Call(adminServiceName+".ReInitFormat", &args, &reply)
 }
 
-// ListLocks - Sends list locks command to remote server via RPC.
-func (rpcClient *AdminRPCClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	args := ListLocksQuery{
-		Bucket:   bucket,
-		Prefix:   prefix,
-		Duration: duration,
-	}
-	var reply []VolumeLockInfo
-
-	err := rpcClient.Call(adminServiceName+".ListLocks", &args, &reply)
-	return reply, err
-}
-
 // ServerInfo - returns the server info of the server to which the RPC call is made.
 func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
 	err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
@@ -147,7 +134,6 @@ func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
 type adminCmdRunner interface {
 	SignalService(s serviceSignal) error
 	ReInitFormat(dryRun bool) error
-	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
 	ServerInfo() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
 	WriteTmpConfig(tmpFileName string, configBytes []byte) error
@@ -244,56 +230,6 @@ func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
 	errs[0] = invokeServiceCmd(cps[0], cmd)
 }
 
-// listPeerLocksInfo - fetch list of locks held on the given bucket,
-// matching prefix held longer than duration from all peer servers.
-func listPeerLocksInfo(peers adminPeers, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// Used to aggregate volume lock information from all nodes.
-	allLocks := make([][]VolumeLockInfo, len(peers))
-	errs := make([]error, len(peers))
-	var wg sync.WaitGroup
-	localPeer := peers[0]
-	remotePeers := peers[1:]
-	for i, remotePeer := range remotePeers {
-		wg.Add(1)
-		go func(idx int, remotePeer adminPeer) {
-			defer wg.Done()
-			// `remotePeers` is right-shifted by one position relative to `peers`
-			allLocks[idx], errs[idx] = remotePeer.cmdRunner.ListLocks(bucket, prefix, duration)
-		}(i+1, remotePeer)
-	}
-	wg.Wait()
-	allLocks[0], errs[0] = localPeer.cmdRunner.ListLocks(bucket, prefix, duration)
-
-	// Summarizing errors received for ListLocks RPC across all
-	// nodes.  N B the possible unavailability of quorum in errors
-	// applies only to distributed setup.
-	errCount, err := reduceErrs(errs, []error{})
-	if err != nil {
-		if errCount >= (len(peers)/2 + 1) {
-			return nil, err
-		}
-		return nil, InsufficientReadQuorum{}
-	}
-
-	// Group lock information across nodes by (bucket, object)
-	// pair. For readability only.
-	paramLockMap := make(map[nsParam][]VolumeLockInfo)
-	for _, nodeLocks := range allLocks {
-		for _, lockInfo := range nodeLocks {
-			param := nsParam{
-				volume: lockInfo.Bucket,
-				path:   lockInfo.Object,
-			}
-			paramLockMap[param] = append(paramLockMap[param], lockInfo)
-		}
-	}
-	groupedLockInfos := []VolumeLockInfo{}
-	for _, volLocks := range paramLockMap {
-		groupedLockInfos = append(groupedLockInfos, volLocks...)
-	}
-	return groupedLockInfos, nil
-}
-
 // uptimeSlice - used to sort uptimes in chronological order.
 type uptimeSlice []struct {
 	err    error

cmd/admin-rpc-client.go.orig

@@ -1,650 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net"
-	"os"
-	"path"
-	"path/filepath"
-	"sort"
-	"sync"
-	"time"
-
-	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/cmd/logger"
-)
-
-const (
-	// Admin service names
-	signalServiceRPC  = "Admin.SignalService"
-	reInitFormatRPC   = "Admin.ReInitFormat"
-	listLocksRPC      = "Admin.ListLocks"
-	serverInfoDataRPC = "Admin.ServerInfoData"
-	getConfigRPC      = "Admin.GetConfig"
-	writeTmpConfigRPC = "Admin.WriteTmpConfig"
-	commitConfigRPC   = "Admin.CommitConfig"
-)
-
-// localAdminClient - represents admin operation to be executed locally.
-type localAdminClient struct {
-}
-
-// remoteAdminClient - represents admin operation to be executed
-// remotely, via RPC.
-type remoteAdminClient struct {
-	*AuthRPCClient
-}
-
-// adminCmdRunner - abstracts local and remote execution of admin
-// commands like service stop and service restart.
-type adminCmdRunner interface {
-	SignalService(s serviceSignal) error
-	ReInitFormat(dryRun bool) error
-	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
-	ServerInfoData() (ServerInfoData, error)
-	GetConfig() ([]byte, error)
-	WriteTmpConfig(tmpFileName string, configBytes []byte) error
-	CommitConfig(tmpFileName string) error
-}
-
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// SignalService - sends a restart or stop signal to the local server
-func (lc localAdminClient) SignalService(s serviceSignal) error {
-	switch s {
-	case serviceRestart, serviceStop:
-		globalServiceSignalCh <- s
-	default:
-		return errUnsupportedSignal
-	}
-	return nil
-}
-
-// ReInitFormat - re-initialize disk format.
-func (lc localAdminClient) ReInitFormat(dryRun bool) error {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	return objectAPI.ReloadFormat(context.Background(), dryRun)
-}
-
-// ListLocks - Fetches lock information from local lock instrumentation.
-func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// check if objectLayer is initialized, if not return.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return nil, errServerNotInitialized
-	}
-	return objectAPI.ListLocks(context.Background(), bucket, prefix, duration)
-}
-
-func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) {
-	switch s {
-	case serviceRestart, serviceStop:
-		reply := AuthRPCReply{}
-		err = rc.Call(signalServiceRPC, &SignalServiceArgs{Sig: s},
-			&reply)
-	default:
-		err = errUnsupportedSignal
-	}
-	return err
-}
-
-// ReInitFormat - re-initialize disk format, remotely.
-func (rc remoteAdminClient) ReInitFormat(dryRun bool) error {
-	reply := AuthRPCReply{}
-	return rc.Call(reInitFormatRPC, &ReInitFormatArgs{
-		DryRun: dryRun,
-	}, &reply)
-}
-
-// ListLocks - Sends list locks command to remote server via RPC.
-func (rc remoteAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	listArgs := ListLocksQuery{
-		Bucket:   bucket,
-		Prefix:   prefix,
-		Duration: duration,
-	}
-	var reply ListLocksReply
-	if err := rc.Call(listLocksRPC, &listArgs, &reply); err != nil {
-		return nil, err
-	}
-	return reply.VolLocks, nil
-}
-
-// ServerInfoData - Returns the server info of this server.
-func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	if globalBootTime.IsZero() {
-		return sid, errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return sid, errServerNotInitialized
-	}
-	storage := objLayer.StorageInfo(context.Background())
-
-	return ServerInfoData{
-		StorageInfo: storage,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			SQSARN:   globalNotificationSys.GetARNList(),
-			Region:   globalServerConfig.GetRegion(),
-		},
-	}, nil
-}
-
-// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	args := AuthRPCArgs{}
-	reply := ServerInfoDataReply{}
-	err := rc.Call(serverInfoDataRPC, &args, &reply)
-	if err != nil {
-		return sid, err
-	}
-
-	return reply.ServerInfoData, nil
-}
-
-// GetConfig - returns config.json of the local server.
-func (lc localAdminClient) GetConfig() ([]byte, error) {
-	if globalServerConfig == nil {
-		return nil, fmt.Errorf("config not present")
-	}
-
-	return json.Marshal(globalServerConfig)
-}
-
-// GetConfig - returns config.json of the remote server.
-func (rc remoteAdminClient) GetConfig() ([]byte, error) {
-	args := AuthRPCArgs{}
-	reply := ConfigReply{}
-	if err := rc.Call(getConfigRPC, &args, &reply); err != nil {
-		return nil, err
-	}
-	return reply.Config, nil
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// the local server.
-func (lc localAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	return writeTmpConfigCommon(tmpFileName, configBytes)
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// a remote node.
-func (rc remoteAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	wArgs := WriteConfigArgs{
-		TmpFileName: tmpFileName,
-		Buf:         configBytes,
-	}
-
-	err := rc.Call(writeTmpConfigRPC, &wArgs, &WriteConfigReply{})
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	return nil
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a local node.
-func (lc localAdminClient) CommitConfig(tmpFileName string) error {
-	configFile := getConfigFile()
-	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
-
-	err := os.Rename(tmpConfigFile, configFile)
-	reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile)
-	reqInfo.AppendTags("configFile", configFile)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
-	logger.LogIf(ctx, err)
-	return err
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a remote node.
-func (rc remoteAdminClient) CommitConfig(tmpFileName string) error {
-	cArgs := CommitConfigArgs{
-		FileName: tmpFileName,
-	}
-	cReply := CommitConfigReply{}
-	err := rc.Call(commitConfigRPC, &cArgs, &cReply)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	return nil
-}
-
-// adminPeer - represents an entity that implements admin API RPCs.
-type adminPeer struct {
-	addr      string
-	cmdRunner adminCmdRunner
-	isLocal   bool
-}
-
-// type alias for a collection of adminPeer.
-type adminPeers []adminPeer
-
-// makeAdminPeers - helper function to construct a collection of adminPeer.
-func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
-	thisPeer := globalMinioAddr
-	if globalMinioHost == "" {
-		// When host is not explicitly provided simply
-		// use the first IPv4.
-		thisPeer = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
-	}
-	adminPeerList = append(adminPeerList, adminPeer{
-		thisPeer,
-		localAdminClient{},
-		true,
-	})
-
-	hostSet := set.CreateStringSet(globalMinioAddr)
-	cred := globalServerConfig.GetCredential()
-	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
-	for _, host := range GetRemotePeers(endpoints) {
-		if hostSet.Contains(host) {
-			continue
-		}
-		hostSet.Add(host)
-		adminPeerList = append(adminPeerList, adminPeer{
-			addr: host,
-			cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
-				accessKey:       cred.AccessKey,
-				secretKey:       cred.SecretKey,
-				serverAddr:      host,
-				serviceEndpoint: serviceEndpoint,
-				secureConn:      globalIsSSL,
-				serviceName:     "Admin",
-			})},
-		})
-	}
-
-	return adminPeerList
-}
-
-// peersReInitFormat - reinitialize remote object layers to new format.
-func peersReInitFormat(peers adminPeers, dryRun bool) error {
-	errs := make([]error, len(peers))
-
-	// Send ReInitFormat RPC call to all nodes.
-	// for local adminPeer this is a no-op.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			if !peer.isLocal {
-				errs[idx] = peer.cmdRunner.ReInitFormat(dryRun)
-			}
-		}(i, peer)
-	}
-	wg.Wait()
-	return nil
-}
-
-// Initialize global adminPeer collection.
-func initGlobalAdminPeers(endpoints EndpointList) {
-	globalAdminPeers = makeAdminPeers(endpoints)
-}
-
-// invokeServiceCmd - Invoke Restart/Stop command.
-func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
-	switch cmd {
-	case serviceRestart, serviceStop:
-		err = cp.cmdRunner.SignalService(cmd)
-	}
-	return err
-}
-
-// sendServiceCmd - Invoke Restart command on remote peers
-// adminPeer followed by on the local peer.
-func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
-	// Send service command like stop or restart to all remote nodes and finally run on local node.
-	errs := make([]error, len(cps))
-	var wg sync.WaitGroup
-	remotePeers := cps[1:]
-	for i := range remotePeers {
-		wg.Add(1)
-		go func(idx int) {
-			defer wg.Done()
-			// we use idx+1 because remotePeers slice is 1 position shifted w.r.t cps
-			errs[idx+1] = invokeServiceCmd(remotePeers[idx], cmd)
-		}(i)
-	}
-	wg.Wait()
-	errs[0] = invokeServiceCmd(cps[0], cmd)
-}
-
-// listPeerLocksInfo - fetch list of locks held on the given bucket,
-// matching prefix held longer than duration from all peer servers.
-func listPeerLocksInfo(peers adminPeers, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// Used to aggregate volume lock information from all nodes.
-	allLocks := make([][]VolumeLockInfo, len(peers))
-	errs := make([]error, len(peers))
-	var wg sync.WaitGroup
-	localPeer := peers[0]
-	remotePeers := peers[1:]
-	for i, remotePeer := range remotePeers {
-		wg.Add(1)
-		go func(idx int, remotePeer adminPeer) {
-			defer wg.Done()
-			// `remotePeers` is right-shifted by one position relative to `peers`
-			allLocks[idx], errs[idx] = remotePeer.cmdRunner.ListLocks(bucket, prefix, duration)
-		}(i+1, remotePeer)
-	}
-	wg.Wait()
-	allLocks[0], errs[0] = localPeer.cmdRunner.ListLocks(bucket, prefix, duration)
-
-	// Summarizing errors received for ListLocks RPC across all
-	// nodes.  N B the possible unavailability of quorum in errors
-	// applies only to distributed setup.
-	errCount, err := reduceErrs(errs, []error{})
-	if err != nil {
-		if errCount >= (len(peers)/2 + 1) {
-			return nil, err
-		}
-		return nil, InsufficientReadQuorum{}
-	}
-
-	// Group lock information across nodes by (bucket, object)
-	// pair. For readability only.
-	paramLockMap := make(map[nsParam][]VolumeLockInfo)
-	for _, nodeLocks := range allLocks {
-		for _, lockInfo := range nodeLocks {
-			param := nsParam{
-				volume: lockInfo.Bucket,
-				path:   lockInfo.Object,
-			}
-			paramLockMap[param] = append(paramLockMap[param], lockInfo)
-		}
-	}
-	groupedLockInfos := []VolumeLockInfo{}
-	for _, volLocks := range paramLockMap {
-		groupedLockInfos = append(groupedLockInfos, volLocks...)
-	}
-	return groupedLockInfos, nil
-}
-
-// uptimeSlice - used to sort uptimes in chronological order.
-type uptimeSlice []struct {
-	err    error
-	uptime time.Duration
-}
-
-func (ts uptimeSlice) Len() int {
-	return len(ts)
-}
-
-func (ts uptimeSlice) Less(i, j int) bool {
-	return ts[i].uptime < ts[j].uptime
-}
-
-func (ts uptimeSlice) Swap(i, j int) {
-	ts[i], ts[j] = ts[j], ts[i]
-}
-
-// getPeerUptimes - returns the uptime since the last time read quorum
-// was established on success. Otherwise returns errXLReadQuorum.
-func getPeerUptimes(peers adminPeers) (time.Duration, error) {
-	// In a single node Erasure or FS backend setup the uptime of
-	// the setup is the uptime of the single minio server
-	// instance.
-	if !globalIsDistXL {
-		return UTCNow().Sub(globalBootTime), nil
-	}
-
-	uptimes := make(uptimeSlice, len(peers))
-
-	// Get up time of all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
-			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Sort uptimes in chronological order.
-	sort.Sort(uptimes)
-
-	// Pick the readQuorum'th uptime in chronological order. i.e,
-	// the time at which read quorum was (re-)established.
-	readQuorum := len(uptimes) / 2
-	validCount := 0
-	latestUptime := time.Duration(0)
-	for _, uptime := range uptimes {
-		if uptime.err != nil {
-			logger.LogIf(context.Background(), uptime.err)
-			continue
-		}
-
-		validCount++
-		if validCount >= readQuorum {
-			latestUptime = uptime.uptime
-			break
-		}
-	}
-
-	// Less than readQuorum "Admin.Uptime" RPC call returned
-	// successfully, so read-quorum unavailable.
-	if validCount < readQuorum {
-		return time.Duration(0), InsufficientReadQuorum{}
-	}
-
-	return latestUptime, nil
-}
-
-// getPeerConfig - Fetches config.json from all nodes in the setup and
-// returns the one that occurs in a majority of them.
-func getPeerConfig(peers adminPeers) ([]byte, error) {
-	if !globalIsDistXL {
-		return peers[0].cmdRunner.GetConfig()
-	}
-
-	errs := make([]error, len(peers))
-	configs := make([][]byte, len(peers))
-
-	// Get config from all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			configs[idx], errs[idx] = peer.cmdRunner.GetConfig()
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Find the maximally occurring config among peers in a
-	// distributed setup.
-
-	serverConfigs := make([]serverConfig, len(peers))
-	for i, configBytes := range configs {
-		if errs[i] != nil {
-			continue
-		}
-
-		// Unmarshal the received config files.
-		err := json.Unmarshal(configBytes, &serverConfigs[i])
-		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peers[i].addr)
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-	}
-
-	configJSON, err := getValidServerConfig(serverConfigs, errs)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return nil, err
-	}
-
-	// Return the config.json that was present quorum or more
-	// number of disks.
-	return json.Marshal(configJSON)
-}
-
-// getValidServerConfig - finds the server config that is present in
-// quorum or more number of servers.
-func getValidServerConfig(serverConfigs []serverConfig, errs []error) (scv serverConfig, e error) {
-	// majority-based quorum
-	quorum := len(serverConfigs)/2 + 1
-
-	// Count the number of disks a config.json was found in.
-	configCounter := make([]int, len(serverConfigs))
-
-	// We group equal serverConfigs by the lowest index of the
-	// same value;  e.g, let us take the following serverConfigs
-	// in a 4-node setup,
-	// serverConfigs == [c1, c2, c1, c1]
-	// configCounter == [3, 1, 0, 0]
-	// c1, c2 are the only distinct values that appear.  c1 is
-	// identified by 0, the lowest index it appears in and c2 is
-	// identified by 1. So, we need to find the number of times
-	// each of these distinct values occur.
-
-	// Invariants:
-
-	// 1. At the beginning of the i-th iteration, the number of
-	// unique configurations seen so far is equal to the number of
-	// non-zero counter values in config[:i].
-
-	// 2. At the beginning of the i-th iteration, the sum of
-	// elements of configCounter[:i] is equal to the number of
-	// non-error configurations seen so far.
-
-	// For each of the serverConfig ...
-	for i := range serverConfigs {
-		// Skip nodes where getConfig failed.
-		if errs[i] != nil {
-			continue
-		}
-		// Check if it is equal to any of the configurations
-		// seen so far. If j == i is reached then we have an
-		// unseen configuration.
-		for j := 0; j <= i; j++ {
-			if j < i && configCounter[j] == 0 {
-				// serverConfigs[j] is known to be
-				// equal to a value that was already
-				// seen. See example above for
-				// clarity.
-				continue
-			} else if j < i && serverConfigs[i].ConfigDiff(&serverConfigs[j]) == "" {
-				// serverConfigs[i] is equal to
-				// serverConfigs[j], update
-				// serverConfigs[j]'s counter since it
-				// is the lower index.
-				configCounter[j]++
-				break
-			} else if j == i {
-				// serverConfigs[i] is equal to no
-				// other value seen before. It is
-				// unique so far.
-				configCounter[i] = 1
-				break
-			} // else invariants specified above are violated.
-		}
-	}
-
-	// We find the maximally occurring server config and check if
-	// there is quorum.
-	var configJSON serverConfig
-	maxOccurrence := 0
-	for i, count := range configCounter {
-		if maxOccurrence < count {
-			maxOccurrence = count
-			configJSON = serverConfigs[i]
-		}
-	}
-
-	// If quorum nodes don't agree.
-	if maxOccurrence < quorum {
-		return scv, errXLWriteQuorum
-	}
-
-	return configJSON, nil
-}
-
-// Write config contents into a temporary file on all nodes.
-func writeTmpConfigPeers(peers adminPeers, tmpFileName string, configBytes []byte) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		err := peers[0].cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		return []error{err}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Write config into temporary file on all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return bytes written and errors (if any) during writing
-	// temporary config file.
-	return errs
-}
-
-// Move config contents from the given temporary file onto config.json
-// on all nodes.
-func commitConfigPeers(peers adminPeers, tmpFileName string) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		return []error{peers[0].cmdRunner.CommitConfig(tmpFileName)}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Rename temporary config file into configDir/config.json on
-	// all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.CommitConfig(tmpFileName)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return errors (if any) received during rename.
-	return errs
-}

cmd/admin-rpc-server.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"context"
 	"path"
-	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
@@ -47,20 +46,6 @@ func (receiver *adminRPCReceiver) SignalService(args *SignalServiceArgs, reply *
 	return receiver.local.SignalService(args.Sig)
 }
 
-// ListLocksQuery - wraps ListLocks API's query values to send over RPC.
-type ListLocksQuery struct {
-	AuthArgs
-	Bucket   string
-	Prefix   string
-	Duration time.Duration
-}
-
-// ListLocks - lists locks held by requests handled by this server instance.
-func (receiver *adminRPCReceiver) ListLocks(args *ListLocksQuery, reply *[]VolumeLockInfo) (err error) {
-	*reply, err = receiver.local.ListLocks(args.Bucket, args.Prefix, args.Duration)
-	return err
-}
-
 // ServerInfo - returns the server info when object layer was initialized on this server.
 func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) (err error) {
 	*reply, err = receiver.local.ServerInfo()

cmd/admin-rpc_test.go

@@ -99,31 +99,6 @@ func testAdminCmdRunnerReInitFormat(t *testing.T, client adminCmdRunner) {
 	}
 }
 
-func testAdminCmdRunnerListLocks(t *testing.T, client adminCmdRunner) {
-	tmpGlobalObjectAPI := globalObjectAPI
-	defer func() {
-		globalObjectAPI = tmpGlobalObjectAPI
-	}()
-
-	testCases := []struct {
-		objectAPI ObjectLayer
-		expectErr bool
-	}{
-		{&DummyObjectLayer{}, false},
-		{nil, true},
-	}
-
-	for i, testCase := range testCases {
-		globalObjectAPI = testCase.objectAPI
-		_, err := client.ListLocks("", "", time.Duration(0))
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
 func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
 	tmpGlobalBootTime := globalBootTime
 	tmpGlobalObjectAPI := globalObjectAPI
@@ -139,10 +114,8 @@ func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
 	}()
 
 	endpoints := new(EndpointList)
-	notificationSys, err := NewNotificationSys(globalServerConfig, *endpoints)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
+
+	notificationSys := NewNotificationSys(globalServerConfig, *endpoints)
 
 	testCases := []struct {
 		bootTime        time.Time
@@ -325,16 +298,6 @@ func TestAdminRPCClientReInitFormat(t *testing.T) {
 	testAdminCmdRunnerReInitFormat(t, rpcClient)
 }
 
-func TestAdminRPCClientListLocks(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerListLocks(t, rpcClient)
-}
-
 func TestAdminRPCClientServerInfo(t *testing.T) {
 	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
 	defer httpServer.Close()

cmd/api-errors.go

@@ -22,8 +22,8 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/coreos/etcd/client"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -40,8 +40,8 @@ type APIErrorResponse struct {
 	XMLName    xml.Name `xml:"Error" json:"-"`
 	Code       string
 	Message    string
-	Key        string
-	BucketName string
+	Key        string `xml:"Key,omitempty" json:"Key,omitempty"`
+	BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"`
 	Resource   string
 	RequestID  string `xml:"RequestId" json:"RequestId"`
 	HostID     string `xml:"HostId" json:"HostId"`
@@ -169,7 +169,6 @@ const (
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
-	ErrPartsSizeUnequal
 	ErrInvalidRequest
 	// Minio storage class error codes
 	ErrInvalidStorageClass
@@ -785,11 +784,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Your metadata headers are not supported.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrPartsSizeUnequal: {
-		Code:           "XMinioPartsSizeUnequal",
-		Description:    "All parts except the last part should be of the same size.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrObjectTampered: {
 		Code:           "XMinioObjectTampered",
 		Description:    errObjectTampered.Error(),
@@ -903,10 +897,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 	// etcd specific errors, a key is always a bucket for us return
 	// ErrNoSuchBucket in such a case.
-	if e, ok := err.(*client.Error); ok {
-		if e.Code == client.ErrorCodeKeyNotFound {
-			return ErrNoSuchBucket
-		}
+	if err == dns.ErrNoEntriesFound {
+		return ErrNoSuchBucket
 	}
 
 	switch err.(type) {
@@ -972,8 +964,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooLarge
 	case UnsupportedMetadata:
 		apiErr = ErrUnsupportedMetadata
-	case PartsSizeUnequal:
-		apiErr = ErrPartsSizeUnequal
 	case BucketPolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
 	case *event.ErrInvalidEventName:
@@ -1014,12 +1004,12 @@ func getAPIError(code APIErrorCode) APIError {
 
 // getErrorResponse gets in standard error and resource value and
 // provides a encodable populated response values
-func getAPIErrorResponse(err APIError, resource string) APIErrorResponse {
+func getAPIErrorResponse(err APIError, resource, requestid string) APIErrorResponse {
 	return APIErrorResponse{
 		Code:      err.Code,
 		Message:   err.Description,
 		Resource:  resource,
-		RequestID: "3L137",
+		RequestID: requestid,
 		HostID:    "3L137",
 	}
 }

cmd/api-headers.go

@@ -34,8 +34,6 @@ func mustGetRequestID(t time.Time) string {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	// Set unique request ID for each reply.
-	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
 	w.Header().Set("Server", globalServerUserAgent)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.

cmd/api-response-multipart.go

@@ -44,7 +44,7 @@ func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err Par
 
 	apiError := getAPIError(toAPIErrorCode(err))
 	// Generate complete multipart error response.
-	errorResponse := getAPIErrorResponse(apiError, r.URL.Path)
+	errorResponse := getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey))
 	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
 	encodedErrorResponse := encodeResponse(cmpErrResp)
 

cmd/api-response.go

@@ -577,7 +577,7 @@ func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *u
 	}
 	apiError := getAPIError(errorCode)
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
+	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
 	encodedErrorResponse := encodeResponse(errorResponse)
 	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
@@ -592,7 +592,7 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, errorCode APIErrorCode
 func writeErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
 	apiError := getAPIError(errorCode)
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
+	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }

cmd/api-router.go

@@ -108,7 +108,7 @@ func registerAPIRouter(router *mux.Router) {
 		// HeadBucket
 		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
 		// PostPolicy
-		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
+		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
 		// DeleteMultipleObjects
 		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
 		// DeleteBucketPolicy

cmd/bucket-handlers-listobjects.go

@@ -54,7 +54,7 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 // NOTE: It is recommended that this API to be used for application development.
 // Minio continues to support ListObjectsV1 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectsV2")
+	ctx := newContext(r, w, "ListObjectsV2")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -123,7 +123,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 // criteria to return a subset of the objects in a bucket.
 //
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectsV1")
+	ctx := newContext(r, w, "ListObjectsV1")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/bucket-handlers.go

@@ -28,9 +28,6 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
-
-	etcd "github.com/coreos/etcd/client"
 
 	"github.com/gorilla/mux"
 
@@ -64,7 +61,7 @@ func initFederatorBackend(objLayer ObjectLayer) {
 		g.Go(func() error {
 			r, gerr := globalDNSConfig.Get(b[index].Name)
 			if gerr != nil {
-				if etcd.IsKeyNotFound(gerr) || gerr == dns.ErrNoEntriesFound {
+				if gerr == dns.ErrNoEntriesFound {
 					return globalDNSConfig.Put(b[index].Name)
 				}
 				return gerr
@@ -89,7 +86,7 @@ func initFederatorBackend(objLayer ObjectLayer) {
 // -------------------------
 // This operation returns bucket location.
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketLocation")
+	ctx := newContext(r, w, "GetBucketLocation")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -105,12 +102,6 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetRLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-	defer bucketLock.RUnlock()
 	getBucketInfo := objectAPI.GetBucketInfo
 	if api.CacheAPI() != nil {
 		getBucketInfo = api.CacheAPI().GetBucketInfo
@@ -143,7 +134,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 // uploads in the response.
 //
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListMultipartUploads")
+	ctx := newContext(r, w, "ListMultipartUploads")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -190,7 +181,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 // This implementation of the GET operation returns a list of all buckets
 // owned by the authenticated sender of the request.
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListBuckets")
+	ctx := newContext(r, w, "ListBuckets")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -211,7 +202,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	var bucketsInfo []BucketInfo
 	if globalDNSConfig != nil {
 		dnsBuckets, err := globalDNSConfig.List()
-		if err != nil && !etcd.IsKeyNotFound(err) && err != dns.ErrNoEntriesFound {
+		if err != nil && err != dns.ErrNoEntriesFound {
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 			return
 		}
@@ -246,7 +237,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 // DeleteMultipleObjectsHandler - deletes multiple objects.
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteMultipleObjects")
+	ctx := newContext(r, w, "DeleteMultipleObjects")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -313,34 +304,24 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var wg = &sync.WaitGroup{} // Allocate a new wait group.
-	var dErrs = make([]error, len(deleteObjects.Objects))
-
-	// Delete all requested objects in parallel.
-	for index, object := range deleteObjects.Objects {
-		wg.Add(1)
-		go func(i int, obj ObjectIdentifier) {
-			defer wg.Done()
-			// If the request is denied access, each item
-			// should be marked as 'AccessDenied'
-			if s3Error == ErrAccessDenied {
-				dErrs[i] = PrefixAccessDenied{
-					Bucket: bucket,
-					Object: obj.ObjectName,
-				}
-				return
-			}
-			deleteObject := objectAPI.DeleteObject
-			if api.CacheAPI() != nil {
-				deleteObject = api.CacheAPI().DeleteObject
-			}
-			dErr := deleteObject(ctx, bucket, obj.ObjectName)
-			if dErr != nil {
-				dErrs[i] = dErr
-			}
-		}(index, object)
+	deleteObject := objectAPI.DeleteObject
+	if api.CacheAPI() != nil {
+		deleteObject = api.CacheAPI().DeleteObject
+	}
+
+	var dErrs = make([]error, len(deleteObjects.Objects))
+	for index, object := range deleteObjects.Objects {
+		// If the request is denied access, each item
+		// should be marked as 'AccessDenied'
+		if s3Error == ErrAccessDenied {
+			dErrs[index] = PrefixAccessDenied{
+				Bucket: bucket,
+				Object: object.ObjectName,
+			}
+			continue
+		}
+		dErrs[index] = deleteObject(ctx, bucket, object.ObjectName)
 	}
-	wg.Wait()
 
 	// Collect deleted objects and errors if any.
 	var deletedObjects []ObjectIdentifier
@@ -400,7 +381,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 // ----------
 // This implementation of the PUT operation creates a new bucket for authenticated request
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucket")
+	ctx := newContext(r, w, "PutBucket")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -432,7 +413,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	if globalDNSConfig != nil {
 		if _, err := globalDNSConfig.Get(bucket); err != nil {
-			if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
+			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
 					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -476,7 +457,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 // This implementation of the POST operation handles object creation with a specified
 // signature policy in multipart/form-data
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PostPolicyBucket")
+	ctx := newContext(r, w, "PostPolicyBucket")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -601,7 +582,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata, err := extractMetadataFromHeader(ctx, formValues)
+	metadata := make(map[string]string)
+	err = extractMetadataFromMap(ctx, formValues, metadata)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
@@ -695,7 +677,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 // have permission to access it. Otherwise, the operation might
 // return responses such as 404 Not Found and 403 Forbidden.
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "HeadBucket")
+	ctx := newContext(r, w, "HeadBucket")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -725,7 +707,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 // DeleteBucketHandler - Delete bucket
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteBucket")
+	ctx := newContext(r, w, "DeleteBucket")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/bucket-notification-handlers.go

@@ -42,7 +42,7 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 // as per http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 // It returns empty configuration if its not set.
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketNotification")
+	ctx := newContext(r, w, "GetBucketNotification")
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
@@ -94,7 +94,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 // PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucketNotification")
+	ctx := newContext(r, w, "PutBucketNotification")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -154,7 +154,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 // ListenBucketNotificationHandler - This HTTP handler sends events to the connected HTTP client.
 // Client should send prefix/suffix object name to match and events to watch as query parameters.
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListenBucketNotification")
+	ctx := newContext(r, w, "ListenBucketNotification")
 
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
@@ -235,7 +235,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
-	if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
+	if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/bucket-policy-handlers.go

@@ -38,7 +38,7 @@ const (
 // PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per
 // https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutBucketPolicy")
+	ctx := newContext(r, w, "PutBucketPolicy")
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -99,7 +99,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 // DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration.
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteBucketPolicy")
+	ctx := newContext(r, w, "DeleteBucketPolicy")
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -135,7 +135,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 // GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration.
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetBucketPolicy")
+	ctx := newContext(r, w, "GetBucketPolicy")
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-policy-handlers_test.go

@@ -437,11 +437,13 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 		if recV4.Code != testCase.expectedRespStatus {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			var expectedPolicy *policy.Policy
+			expectedPolicy, err = policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}
-			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			var gotPolicy *policy.Policy
+			gotPolicy, err = policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}

cmd/common-main.go

@@ -26,7 +26,7 @@ import (
 	"strings"
 	"time"
 
-	etcd "github.com/coreos/etcd/client"
+	etcd "github.com/coreos/etcd/clientv3"
 
 	"github.com/minio/cli"
 	"github.com/minio/minio/cmd/logger"
@@ -51,17 +51,19 @@ func checkUpdate(mode string) {
 // Initialize and load config from remote etcd or local config directory
 func initConfig() {
 	if globalEtcdClient != nil {
-		kapi := etcd.NewKeysAPI(globalEtcdClient)
 		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
-		_, err := kapi.Get(ctx, getConfigFile(), nil)
+		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
 		cancel()
-		if err == nil {
-			logger.FatalIf(migrateConfig(), "Config migration failed.")
-			logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
+		// This means there are no entries in etcd with config file
+		// So create a new config
+		if err == nil && resp.Count == 0 {
+			logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+			logger.Info("Created minio configuration file successfully at %v", globalEtcdClient.Endpoints())
 		} else {
-			if etcd.IsKeyNotFound(err) {
-				logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
-				logger.Info("Created minio configuration file successfully at %v", globalEtcdClient.Endpoints())
+			// This means there is an entry in etcd, update it if required and proceed
+			if err == nil && resp.Count > 0 {
+				logger.FatalIf(migrateConfig(), "Config migration failed.")
+				logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
 			} else {
 				logger.FatalIf(err, "Unable to load config version: '%s'.", serverConfigVersion)
 			}
@@ -79,6 +81,20 @@ func initConfig() {
 	}
 }
 
+// Load logger targets based on user's configuration
+func loadLoggers() {
+	if globalServerConfig.Logger.Console.Enabled {
+		// Enable console logging
+		logger.AddTarget(logger.NewConsole())
+	}
+	for _, l := range globalServerConfig.Logger.HTTP {
+		if l.Enabled {
+			// Enable http logging
+			logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
+		}
+	}
+}
+
 func handleCommonCmdArgs(ctx *cli.Context) {
 
 	var configDir string
@@ -155,8 +171,9 @@ func handleCommonEnvVars() {
 		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
 		var err error
 		globalEtcdClient, err = etcd.New(etcd.Config{
-			Endpoints: etcdEndpoints,
-			Transport: NewCustomHTTPTransport(),
+			Endpoints:         etcdEndpoints,
+			DialTimeout:       defaultDialTimeout,
+			DialKeepAliveTime: defaultDialKeepAlive,
 		})
 		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
 	}

cmd/config-current.go

@@ -17,11 +17,13 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"reflect"
 	"sync"
 
+	"github.com/miekg/dns"
 	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/auth"
@@ -39,9 +41,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "26"
+const serverConfigVersion = "27"
 
-type serverConfig = serverConfigV26
+type serverConfig = serverConfigV27
 
 var (
 	// globalServerConfig server config.
@@ -128,6 +130,84 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 	return s.Cache
 }
 
+func (s *serverConfig) Validate() error {
+	if s.Version != serverConfigVersion {
+		return fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, s.Version)
+	}
+
+	// Validate credential fields only when
+	// they are not set via the environment
+	// Error out if global is env credential is not set and config has invalid credential
+	if !globalIsEnvCreds && !s.Credential.IsValid() {
+		return errors.New("invalid credential in config file")
+	}
+
+	// Region: nothing to validate
+	// Browser, Worm, Cache and StorageClass values are already validated during json unmarshal
+
+	if s.Domain != "" {
+		if _, ok := dns.IsDomainName(s.Domain); !ok {
+			return errors.New("invalid domain name")
+		}
+	}
+
+	for _, v := range s.Notify.AMQP {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("amqp: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.Elasticsearch {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("elasticsearch: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.Kafka {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("kafka: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.MQTT {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("mqtt: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.MySQL {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("mysql: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.NATS {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("nats: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.PostgreSQL {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("postgreSQL: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.Redis {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("redis: %s", err.Error())
+		}
+	}
+
+	for _, v := range s.Notify.Webhook {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("webhook: %s", err.Error())
+		}
+	}
+
+	return nil
+}
+
 // Save config file to corresponding backend
 func Save(configFile string, data interface{}) error {
 	return quick.SaveConfig(data, configFile, globalEtcdClient)
@@ -180,6 +260,8 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "MySQL Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
 		return "MQTT Notification configuration differs"
+	case !reflect.DeepEqual(s.Logger, t.Logger):
+		return "Logger configuration differs"
 	case reflect.DeepEqual(s, t):
 		return ""
 	default:
@@ -235,6 +317,13 @@ func newServerConfig() *serverConfig {
 	srvCfg.Cache.Exclude = make([]string, 0)
 	srvCfg.Cache.Expiry = globalCacheExpiry
 	srvCfg.Cache.MaxUse = globalCacheMaxUse
+
+	// Console logging is on by default
+	srvCfg.Logger.Console.Enabled = true
+	// Create an example of HTTP logger
+	srvCfg.Logger.HTTP = make(map[string]loggerHTTP)
+	srvCfg.Logger.HTTP["target1"] = loggerHTTP{Endpoint: "https://username:password@example.com/api"}
+
 	return srvCfg
 }
 
@@ -317,15 +406,8 @@ func getValidConfig() (*serverConfig, error) {
 		return nil, err
 	}
 
-	if srvCfg.Version != serverConfigVersion {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
-	}
-
-	// Validate credential fields only when
-	// they are not set via the environment
-	// Error out if global is env credential is not set and config has invalid credential
-	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
-		return nil, errors.New("invalid credential in config file " + getConfigFile())
+	if err = srvCfg.Validate(); err != nil {
+		return nil, err
 	}
 
 	return srvCfg, nil
@@ -402,17 +484,19 @@ func loadConfig() error {
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
+func getNotificationTargets(config *serverConfig) *event.TargetList {
 	targetList := event.NewTargetList()
 
 	for id, args := range config.Notify.AMQP {
 		if args.Enable {
 			newTarget, err := target.NewAMQPTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -421,10 +505,14 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewElasticsearchTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 		}
 	}
@@ -433,10 +521,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewKafkaTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -445,10 +535,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewMQTTTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -457,10 +549,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewMySQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -469,10 +563,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewNATSTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -481,10 +577,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewPostgreSQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -493,10 +591,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewRedisTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -505,10 +605,11 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget := target.NewWebhookTarget(id, args)
 			if err := targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
 
-	return targetList, nil
+	return targetList
 }

cmd/config-current_test.go

@@ -174,55 +174,55 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
 
 		// Test 11 - Test AMQP
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
 
 		// Test 12 - Test NATS
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
 		// Test 13 - Test ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
 
 		// Test 14 - Test Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
 		// Test 15 - Test PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
 		// Test 16 - Test Kafka
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
 
 		// Test 17 - Test Webhook
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
 
 		// Test 18 - Test MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
 		// Test 19 - Test Format for MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
 		// Test 20 - Test valid Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
 		// Test 21 - Test Format for PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
 		// Test 22 - Test valid Format for PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
 		// Test 23 - Test Format for ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
 
 		// Test 24 - Test valid Format for ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
 
 		// Test 25 - Test Format for Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
 
 		// Test 26 - Test valid Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
 
 		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
 	}
 
 	for i, testCase := range testCases {
@@ -319,6 +319,18 @@ func TestConfigDiff(t *testing.T) {
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
 			"MQTT Notification configuration differs",
 		},
+		// 16
+		{
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
+			}},
+			&serverConfig{Logger: loggerConfig{
+				Console: loggerConsole{Enabled: true},
+				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
+			}},
+			"Logger configuration differs",
+		},
 	}
 
 	for i, testCase := range testCases {

cmd/config-migrate.go

@@ -21,9 +21,9 @@ import (
 	"os"
 	"path/filepath"
 
-	etcd "github.com/coreos/etcd/client"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	xnet "github.com/minio/minio/pkg/net"
@@ -188,6 +188,11 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
+	case "26":
+		if err = migrateV26ToV27(); err != nil {
+			return err
+		}
+		fallthrough
 	case serverConfigVersion:
 		// No migration needed. this always points to current version.
 		err = nil
@@ -201,7 +206,7 @@ func purgeV1() error {
 
 	cv1 := &configV1{}
 	_, err := Load(configFile, cv1)
-	if os.IsNotExist(err) || etcd.IsKeyNotFound(err) {
+	if os.IsNotExist(err) || err == dns.ErrNoEntriesFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@@ -2317,3 +2322,36 @@ func migrateV25ToV26() error {
 	logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
 	return nil
 }
+
+func migrateV26ToV27() error {
+	configFile := getConfigFile()
+
+	// config V27 is backward compatible with V26, load the old
+	// config file in serverConfigV27 struct and put some examples
+	// in the new `logger` field
+	srvConfig := &serverConfigV27{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+
+	if srvConfig.Version != "26" {
+		return nil
+	}
+
+	srvConfig.Version = "27"
+	// Enable console logging by default to avoid breaking users
+	// current deployments
+	srvConfig.Logger.Console.Enabled = true
+	srvConfig.Logger.HTTP = make(map[string]loggerHTTP)
+	srvConfig.Logger.HTTP["1"] = loggerHTTP{}
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "26", "27")
+	return nil
+}

cmd/config-migrate_test.go

@@ -134,10 +134,25 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV21ToV22(); err != nil {
 		t.Fatal("migrate v21 to v22 should succeed when no config file is found")
 	}
+	if err := migrateV22ToV23(); err != nil {
+		t.Fatal("migrate v22 to v23 should succeed when no config file is found")
+	}
+	if err := migrateV23ToV24(); err != nil {
+		t.Fatal("migrate v23 to v24 should succeed when no config file is found")
+	}
+	if err := migrateV24ToV25(); err != nil {
+		t.Fatal("migrate v24 to v25 should succeed when no config file is found")
+	}
+	if err := migrateV25ToV26(); err != nil {
+		t.Fatal("migrate v25 to v26 should succeed when no config file is found")
+	}
+	if err := migrateV26ToV27(); err != nil {
+		t.Fatal("migrate v26 to v27 should succeed when no config file is found")
+	}
 }
 
-// Test if a config migration from v2 to v23 is successfully done
-func TestServerConfigMigrateV2toV23(t *testing.T) {
+// Test if a config migration from v2 to v27 is successfully done
+func TestServerConfigMigrateV2toV27(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
@@ -272,6 +287,18 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV22ToV23(); err == nil {
 		t.Fatal("migrateConfigV22ToV23() should fail with a corrupted json")
 	}
+	if err := migrateV23ToV24(); err == nil {
+		t.Fatal("migrateConfigV23ToV24() should fail with a corrupted json")
+	}
+	if err := migrateV24ToV25(); err == nil {
+		t.Fatal("migrateConfigV24ToV25() should fail with a corrupted json")
+	}
+	if err := migrateV25ToV26(); err == nil {
+		t.Fatal("migrateConfigV25ToV26() should fail with a corrupted json")
+	}
+	if err := migrateV26ToV27(); err == nil {
+		t.Fatal("migrateConfigV26ToV27() should fail with a corrupted json")
+	}
 }
 
 // Test if all migrate code returns error with corrupted config files

cmd/config-versions.go

@@ -657,9 +657,6 @@ type serverConfigV25 struct {
 
 // serverConfigV26 is just like version '25', stores additionally
 // cache max use value in 'CacheConfig'.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV26 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -681,3 +678,47 @@ type serverConfigV26 struct {
 	// Notification queue configuration.
 	Notify notifier `json:"notify"`
 }
+
+type loggerConsole struct {
+	Enabled bool `json:"enabled"`
+}
+
+type loggerHTTP struct {
+	Enabled  bool   `json:"enabled"`
+	Endpoint string `json:"endpoint"`
+}
+
+type loggerConfig struct {
+	Console loggerConsole         `json:"console"`
+	HTTP    map[string]loggerHTTP `json:"http"`
+}
+
+// serverConfigV27 is just like version '26', stores additionally
+// the logger field
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV27 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+}

cmd/crypto/error.go

@@ -16,8 +16,49 @@ package crypto
 
 import "errors"
 
+// Error is the generic type for any error happening during decrypting
+// an object. It indicates that the object itself or its metadata was
+// modified accidentally or maliciously.
+type Error struct{ msg string }
+
+func (e Error) Error() string { return e.msg }
+
 var (
 	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
 	// is not supported.
 	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+
+	// ErrInvalidCustomerAlgorithm indicates that the specified SSE-C algorithm
+	// is not supported.
+	ErrInvalidCustomerAlgorithm = errors.New("The SSE-C algorithm is not supported")
+
+	// ErrMissingCustomerKey indicates that the HTTP headers contains no SSE-C client key.
+	ErrMissingCustomerKey = errors.New("The SSE-C request is missing the customer key")
+
+	// ErrMissingCustomerKeyMD5 indicates that the HTTP headers contains no SSE-C client key
+	// MD5 checksum.
+	ErrMissingCustomerKeyMD5 = errors.New("The SSE-C request is missing the customer key MD5")
+
+	// ErrInvalidCustomerKey indicates that the SSE-C client key is not valid - e.g. not a
+	// base64-encoded string or not 256 bits long.
+	ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
+
+	// ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
+	// computed MD5 sum. This means that the client provided either the wrong key for
+	// a certain MD5 checksum or the wrong MD5 for a certain key.
+	ErrCustomerKeyMD5Mismatch = errors.New("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
+)
+
+var (
+	errMissingInternalIV            = Error{"The object metadata is missing the internal encryption IV"}
+	errMissingInternalSealAlgorithm = Error{"The object metadata is missing the internal seal algorithm"}
+
+	errInvalidInternalIV            = Error{"The internal encryption IV is malformed"}
+	errInvalidInternalSealAlgorithm = Error{"The internal seal algorithm is invalid and not supported"}
+)
+
+var (
+	// errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able
+	// to produce enough random data. This is fatal error and should cause a panic.
+	errOutOfEntropy = errors.New("Unable to read enough randomness from the system")
 )

cmd/crypto/header.go

@@ -15,12 +15,43 @@
 package crypto
 
 import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
 	"net/http"
 )
 
 // SSEHeader is the general AWS SSE HTTP header key.
 const SSEHeader = "X-Amz-Server-Side-Encryption"
 
+const (
+	// SSECAlgorithm is the HTTP header key referencing
+	// the SSE-C algorithm.
+	SSECAlgorithm = SSEHeader + "-Customer-Algorithm"
+
+	// SSECKey is the HTTP header key referencing the
+	// SSE-C client-provided key..
+	SSECKey = SSEHeader + "-Customer-Key"
+
+	// SSECKeyMD5 is the HTTP header key referencing
+	// the MD5 sum of the client-provided key.
+	SSECKeyMD5 = SSEHeader + "-Customer-Key-Md5"
+)
+
+const (
+	// SSECopyAlgorithm is the HTTP header key referencing
+	// the SSE-C algorithm for SSE-C copy requests.
+	SSECopyAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
+
+	// SSECopyKey is the HTTP header key referencing the SSE-C
+	// client-provided key for SSE-C copy requests.
+	SSECopyKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
+
+	// SSECopyKeyMD5 is the HTTP header key referencing the
+	// MD5 sum of the client key for SSE-C copy requests.
+	SSECopyKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
+)
+
 // SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
 // For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
 // For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
@@ -39,11 +70,107 @@ func (s3) IsRequested(h http.Header) bool {
 	return ok
 }
 
-// Parse parses the SSE-S3 related HTTP headers and checks
+// ParseHTTP parses the SSE-S3 related HTTP headers and checks
 // whether they contain valid values.
-func (s3) Parse(h http.Header) (err error) {
+func (s3) ParseHTTP(h http.Header) (err error) {
 	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
 		err = ErrInvalidEncryptionMethod
 	}
 	return
 }
+
+var (
+	// SSEC represents AWS SSE-C. It provides functionality to handle
+	// SSE-C requests.
+	SSEC = ssec{}
+
+	// SSECopy represents AWS SSE-C for copy requests. It provides
+	// functionality to handle SSE-C copy requests.
+	SSECopy = ssecCopy{}
+)
+
+type ssec struct{}
+type ssecCopy struct{}
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C header. SSE-C copy headers are ignored.
+func (ssec) IsRequested(h http.Header) bool {
+	if _, ok := h[SSECAlgorithm]; ok {
+		return true
+	}
+	if _, ok := h[SSECKey]; ok {
+		return true
+	}
+	if _, ok := h[SSECKeyMD5]; ok {
+		return true
+	}
+	return false
+}
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C copy header. Regular SSE-C headers
+// are ignored.
+func (ssecCopy) IsRequested(h http.Header) bool {
+	if _, ok := h[SSECopyAlgorithm]; ok {
+		return true
+	}
+	if _, ok := h[SSECopyKey]; ok {
+		return true
+	}
+	if _, ok := h[SSECopyKeyMD5]; ok {
+		return true
+	}
+	return false
+}
+
+// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
+// on success. SSE-C copy headers are ignored.
+func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
+	defer h.Del(SSECKey) // remove SSE-C key from headers after parsing
+	if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
+		return key, ErrInvalidCustomerAlgorithm
+	}
+	if h.Get(SSECKey) == "" {
+		return key, ErrMissingCustomerKey
+	}
+	if h.Get(SSECKeyMD5) == "" {
+		return key, ErrMissingCustomerKeyMD5
+	}
+
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECKey))
+	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+		return key, ErrInvalidCustomerKey
+	}
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECKeyMD5))
+	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+		return key, ErrCustomerKeyMD5Mismatch
+	}
+	copy(key[:], clientKey)
+	return key, nil
+}
+
+// ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
+// on success. Regular SSE-C headers are ignored.
+func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
+	defer h.Del(SSECopyKey) // remove SSE-C copy key of source object from headers after parsing
+	if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
+		return key, ErrInvalidCustomerAlgorithm
+	}
+	if h.Get(SSECopyKey) == "" {
+		return key, ErrMissingCustomerKey
+	}
+	if h.Get(SSECopyKeyMD5) == "" {
+		return key, ErrMissingCustomerKeyMD5
+	}
+
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKey))
+	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+		return key, ErrInvalidCustomerKey
+	}
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKeyMD5))
+	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+		return key, ErrCustomerKeyMD5Mismatch
+	}
+	copy(key[:], clientKey)
+	return key, nil
+}

cmd/crypto/header_test.go

@@ -19,7 +19,7 @@ import (
 	"testing"
 )
 
-var isRequestedTests = []struct {
+var s3IsRequestedTests = []struct {
 	Header   http.Header
 	Expected bool
 }{
@@ -30,14 +30,14 @@ var isRequestedTests = []struct {
 }
 
 func TestS3IsRequested(t *testing.T) {
-	for i, test := range isRequestedTests {
+	for i, test := range s3IsRequestedTests {
 		if got := S3.IsRequested(test.Header); got != test.Expected {
 			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
 }
 
-var parseTests = []struct {
+var s3ParseTests = []struct {
 	Header      http.Header
 	ExpectedErr error
 }{
@@ -48,9 +48,257 @@ var parseTests = []struct {
 }
 
 func TestS3Parse(t *testing.T) {
-	for i, test := range parseTests {
-		if err := S3.Parse(test.Header); err != test.ExpectedErr {
+	for i, test := range s3ParseTests {
+		if err := S3.ParseHTTP(test.Header); err != test.ExpectedErr {
 			t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
 		}
 	}
 }
+
+var ssecIsRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{}, Expected: false},                                                                                                     // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true},                                 // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true},                 // 3
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{""},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{""},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{""},
+		},
+		Expected: true,
+	}, // 4
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: true,
+	}, // 5
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: false,
+	}, // 6
+}
+
+func TestSSECIsRequested(t *testing.T) {
+	for i, test := range ssecIsRequestedTests {
+		if got := SSEC.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var ssecCopyIsRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{}, Expected: false},                                                                                                                 // 0
+	{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true},                                 // 1
+	{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
+	{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true},                 // 3
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{""},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{""},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{""},
+		},
+		Expected: true,
+	}, // 4
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: true,
+	}, // 5
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Expected: false,
+	}, // 6
+}
+
+func TestSSECopyIsRequested(t *testing.T) {
+	for i, test := range ssecCopyIsRequestedTests {
+		if got := SSECopy.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var ssecParseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: nil, // 0
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerAlgorithm, // 1
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{""}, // no client key
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrMissingCustomerKey, // 2
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerKey, // 3
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{""}, // no key MD5
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5, // 4
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6
+	},
+}
+
+func TestSSECParse(t *testing.T) {
+	var zeroKey [32]byte
+	for i, test := range ssecParseTests {
+		key, err := SSEC.ParseHTTP(test.Header)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+
+		if err != nil && key != zeroKey {
+			t.Errorf("Test %d: parsing failed and client key is not zero key", i)
+		}
+		if err == nil && key == zeroKey {
+			t.Errorf("Test %d: parsed client key is zero key", i)
+		}
+		if _, ok := test.Header[SSECKey]; ok {
+			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
+		}
+	}
+}
+
+var ssecCopyParseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: nil, // 0
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerAlgorithm, // 1
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{""}, // no client key
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrMissingCustomerKey, // 2
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrInvalidCustomerKey, // 3
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{""}, // no key MD5
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5, // 4
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5
+	},
+	{
+		Header: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5
+		},
+		ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6
+	},
+}
+
+func TestSSECopyParse(t *testing.T) {
+	var zeroKey [32]byte
+	for i, test := range ssecCopyParseTests {
+		key, err := SSECopy.ParseHTTP(test.Header)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+
+		if err != nil && key != zeroKey {
+			t.Errorf("Test %d: parsing failed and client key is not zero key", i)
+		}
+		if err == nil && key == zeroKey {
+			t.Errorf("Test %d: parsed client key is zero key", i)
+		}
+		if _, ok := test.Header[SSECKey]; ok {
+			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
+		}
+	}
+}

cmd/crypto/key.go

@@ -21,8 +21,9 @@ import (
 	"crypto/rand"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
-	"path/filepath"
+	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	sha256 "github.com/minio/sha256-simd"
@@ -34,50 +35,99 @@ import (
 type ObjectKey [32]byte
 
 // GenerateKey generates a unique ObjectKey from a 256 bit external key
-// and a source of randomness. If random is nil the default PRNG of system
-// (crypto/rand) is used.
+// and a source of randomness. If random is nil the default PRNG of the
+// system (crypto/rand) is used.
 func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
 	if random == nil {
 		random = rand.Reader
 	}
 	var nonce [32]byte
 	if _, err := io.ReadFull(random, nonce[:]); err != nil {
-		logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
 	}
 	sha := sha256.New()
 	sha.Write(extKey[:])
 	sha.Write(nonce[:])
 	sha.Sum(key[:0])
-	return
+	return key
+}
+
+// GenerateIV generates a new random 256 bit IV from the provided source
+// of randomness. If random is nil the default PRNG of the system
+// (crypto/rand) is used.
+func GenerateIV(random io.Reader) (iv [32]byte) {
+	if random == nil {
+		random = rand.Reader
+	}
+	if _, err := io.ReadFull(random, iv[:]); err != nil {
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
+	}
+	return iv
+}
+
+// SealedKey represents a sealed object key. It can be stored
+// at an untrusted location.
+type SealedKey struct {
+	Key       [64]byte // The encrypted and authenticted object-key.
+	IV        [32]byte // The random IV used to encrypt the object-key.
+	Algorithm string   // The sealing algorithm used to encrypt the object key.
 }
 
 // Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
-// key is also cryptographically bound to the object's path (bucket/object).
-func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
-	var sealedKey bytes.Buffer
+// key is also cryptographically bound to the object's path (bucket/object) and the
+// domain (SSE-C or SSE-S3).
+func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) SealedKey {
+	var (
+		sealingKey   [32]byte
+		encryptedKey bytes.Buffer
+	)
 	mac := hmac.New(sha256.New, extKey[:])
 	mac.Write(iv[:])
-	mac.Write([]byte(filepath.Join(bucket, object)))
+	mac.Write([]byte(domain))
+	mac.Write([]byte(SealAlgorithm))
+	mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object'
+	mac.Sum(sealingKey[:0])
 
-	if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
+	if n, err := sio.Encrypt(&encryptedKey, bytes.NewReader(key[:]), sio.Config{Key: sealingKey[:]}); n != 64 || err != nil {
 		logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
 	}
-	return sealedKey.Bytes()
+	sealedKey := SealedKey{
+		IV:        iv,
+		Algorithm: SealAlgorithm,
+	}
+	copy(sealedKey.Key[:], encryptedKey.Bytes())
+	return sealedKey
 }
 
-// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
-// is cryptographically bound to the object's path the same bucket/object as during sealing
+// Unseal decrypts a sealed key using the 256 bit external key. Since the sealed key
+// may be cryptographically bound to the object's path the same bucket/object as during sealing
 // must be provided. On success the ObjectKey contains the decrypted sealed key.
-func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
-	var unsealedKey bytes.Buffer
-	mac := hmac.New(sha256.New, extKey[:])
-	mac.Write(iv[:])
-	mac.Write([]byte(filepath.Join(bucket, object)))
+func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error {
+	var (
+		unsealConfig sio.Config
+		decryptedKey bytes.Buffer
+	)
+	switch sealedKey.Algorithm {
+	default:
+		return Error{fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm)}
+	case SealAlgorithm:
+		mac := hmac.New(sha256.New, extKey[:])
+		mac.Write(sealedKey.IV[:])
+		mac.Write([]byte(domain))
+		mac.Write([]byte(SealAlgorithm))
+		mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object'
+		unsealConfig = sio.Config{MinVersion: sio.Version20, Key: mac.Sum(nil)}
+	case InsecureSealAlgorithm:
+		sha := sha256.New()
+		sha.Write(extKey[:])
+		sha.Write(sealedKey.IV[:])
+		unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)}
+	}
 
-	if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
+	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
 		return err // TODO(aead): upgrade sio to use sio.Error
 	}
-	copy(key[:], unsealedKey.Bytes())
+	copy(key[:], decryptedKey.Bytes())
 	return nil
 }
 
@@ -89,5 +139,5 @@ func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
 	mac := hmac.New(sha256.New, key[:])
 	mac.Write(bin[:])
 	mac.Sum(partKey[:0])
-	return
+	return partKey
 }

cmd/crypto/key_test.go

@@ -20,6 +20,8 @@ import (
 	"encoding/hex"
 	"io"
 	"testing"
+
+	"github.com/minio/minio/cmd/logger"
 )
 
 var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
@@ -37,14 +39,18 @@ var generateKeyTests = []struct {
 	Random     io.Reader
 	ShouldPass bool
 }{
-	{ExtKey: [32]byte{}, Random: nil, ShouldPass: true},             // 0
-	{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true},     // 1
-	{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
-	// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
+	{ExtKey: [32]byte{}, Random: nil, ShouldPass: true},              // 0
+	{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true},      // 1
+	{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true},  // 2
+	{ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3
 }
 
 func TestGenerateKey(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+
 	for i, test := range generateKeyTests {
+		i, test := i, test
 		func() {
 			defer recoverTest(i, test.ShouldPass, t)
 			key := GenerateKey(test.ExtKey, test.Random)
@@ -55,38 +61,64 @@ func TestGenerateKey(t *testing.T) {
 	}
 }
 
-var sealUnsealKeyTests = []struct {
-	SealExtKey, SealIV     [32]byte
-	SealBucket, SealObject string
+var generateIVTests = []struct {
+	Random     io.Reader
+	ShouldPass bool
+}{
+	{Random: nil, ShouldPass: true},              // 0
+	{Random: rand.Reader, ShouldPass: true},      // 1
+	{Random: shortRandom(32), ShouldPass: true},  // 2
+	{Random: shortRandom(31), ShouldPass: false}, // 3
+}
 
-	UnsealExtKey, UnsealIV     [32]byte
-	UnsealBucket, UnsealObject string
+func TestGenerateIV(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+
+	for i, test := range generateIVTests {
+		i, test := i, test
+		func() {
+			defer recoverTest(i, test.ShouldPass, t)
+			iv := GenerateIV(test.Random)
+			if iv == [32]byte{} {
+				t.Errorf("Test %d: generated IV is zero IV", i) // check that we generate random and unique IV
+			}
+		}()
+	}
+}
+
+var sealUnsealKeyTests = []struct {
+	SealExtKey, SealIV                 [32]byte
+	SealDomain, SealBucket, SealObject string
+
+	UnsealExtKey                             [32]byte
+	UnsealDomain, UnsealBucket, UnsealObject string
 
 	ShouldPass bool
 }{
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object",
 		ShouldPass: true,
 	}, // 0
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{1}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different ext-key
 		ShouldPass: false,
 	}, // 1
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-S3", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different domain
 		ShouldPass: false,
 	}, // 2
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "Bucket", UnsealObject: "object", // different bucket
 		ShouldPass: false,
 	}, // 3
 	{
-		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
-		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "Object", // different object
 		ShouldPass: false,
 	}, // 4
 }
@@ -94,13 +126,22 @@ var sealUnsealKeyTests = []struct {
 func TestSealUnsealKey(t *testing.T) {
 	for i, test := range sealUnsealKeyTests {
 		key := GenerateKey(test.SealExtKey, rand.Reader)
-		sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
-		if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
+		sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealDomain, test.SealBucket, test.SealObject)
+		if err := key.Unseal(test.UnsealExtKey, sealedKey, test.UnsealDomain, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
 			t.Errorf("Test %d should fail but passed successfully", i)
 		} else if err != nil && test.ShouldPass {
 			t.Errorf("Test %d should pass put failed: %v", i, err)
 		}
 	}
+
+	// Test legacy InsecureSealAlgorithm
+	var extKey, iv [32]byte
+	key := GenerateKey(extKey, rand.Reader)
+	sealedKey := key.Seal(extKey, iv, "SSE-S3", "bucket", "object")
+	sealedKey.Algorithm = InsecureSealAlgorithm
+	if err := key.Unseal(extKey, sealedKey, "SSE-S3", "bucket", "object"); err == nil {
+		t.Errorf("'%s' test succeeded but it should fail because the legacy algorithm was used", sealedKey.Algorithm)
+	}
 }
 
 var derivePartKeyTest = []struct {

cmd/crypto/kms.go

@@ -0,0 +1,138 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+// Context is a list of key-value pairs cryptographically
+// associated with a certain object.
+type Context map[string]string
+
+// WriteTo writes the context in a canonical from to w.
+// It returns the number of bytes and the first error
+// encounter during writing to w, if any.
+//
+// WriteTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+func (c Context) WriteTo(w io.Writer) (n int64, err error) {
+	sortedKeys := make(sort.StringSlice, 0, len(c))
+	for k := range c {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Sort(sortedKeys)
+
+	nn, err := io.WriteString(w, "{")
+	if err != nil {
+		return n + int64(nn), err
+	}
+	n += int64(nn)
+	for i, k := range sortedKeys {
+		s := fmt.Sprintf("\"%s\":\"%s\",", k, c[k])
+		if i == len(sortedKeys)-1 {
+			s = s[:len(s)-1] // remove last ','
+		}
+
+		nn, err = io.WriteString(w, s)
+		if err != nil {
+			return n + int64(nn), err
+		}
+		n += int64(nn)
+	}
+	nn, err = io.WriteString(w, "}")
+	return n + int64(nn), err
+}
+
+// KMS represents an active and authenticted connection
+// to a Key-Management-Service. It supports generating
+// data key generation and unsealing of KMS-generated
+// data keys.
+type KMS interface {
+	// GenerateKey generates a new random data key using
+	// the master key referenced by the keyID. It returns
+	// the plaintext key and the sealed plaintext key
+	// on success.
+	//
+	// The context is cryptographically bound to the
+	// generated key. The same context must be provided
+	// again to unseal the generated key.
+	GenerateKey(keyID string, context Context) (key [32]byte, sealedKey []byte, err error)
+
+	// UnsealKey unseals the sealedKey using the master key
+	// referenced by the keyID. The provided context must
+	// match the context used to generate the sealed key.
+	UnsealKey(keyID string, sealedKey []byte, context Context) (key [32]byte, err error)
+}
+
+type masterKeyKMS struct {
+	masterKey [32]byte
+}
+
+// NewKMS returns a basic KMS implementation from a single 256 bit master key.
+//
+// The KMS accepts any keyID but binds the keyID and context cryptographically
+// to the generated keys.
+func NewKMS(key [32]byte) KMS { return &masterKeyKMS{masterKey: key} }
+
+func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
+	if _, err = io.ReadFull(rand.Reader, key[:]); err != nil {
+		logger.CriticalIf(context.Background(), errOutOfEntropy)
+	}
+
+	var (
+		buffer     bytes.Buffer
+		derivedKey = kms.deriveKey(keyID, ctx)
+	)
+	if n, err := sio.Encrypt(&buffer, bytes.NewReader(key[:]), sio.Config{Key: derivedKey[:]}); err != nil || n != 64 {
+		logger.CriticalIf(context.Background(), errors.New("KMS: unable to encrypt data key"))
+	}
+	sealedKey = buffer.Bytes()
+	return key, sealedKey, nil
+}
+
+func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
+	var (
+		buffer     bytes.Buffer
+		derivedKey = kms.deriveKey(keyID, ctx)
+	)
+	if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
+		return key, err // TODO(aead): upgrade sio to use sio.Error
+	}
+	copy(key[:], buffer.Bytes())
+	return key, nil
+}
+
+func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte) {
+	if context == nil {
+		context = Context{}
+	}
+	mac := hmac.New(sha256.New, kms.masterKey[:])
+	mac.Write([]byte(keyID))
+	context.WriteTo(mac)
+	mac.Sum(key[:0])
+	return key
+}

cmd/crypto/kms_test.go

@@ -0,0 +1,84 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"path"
+	"strings"
+	"testing"
+)
+
+var masterKeyKMSTests = []struct {
+	GenKeyID, UnsealKeyID     string
+	GenContext, UnsealContext Context
+
+	ShouldFail bool
+}{
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{}, UnsealContext: nil, ShouldFail: false},                                                                                     // 0
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: false},                                                               // 1
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"bucket": "object"}, ShouldFail: false},                           // 2
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: false}, // 3
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "0", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: false},                       // 4
+
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7e", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: true},                                                               // 5
+	{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"Bucket": "object"}, ShouldFail: true},                           // 6
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "Object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: true}, // 7
+	{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "1", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: true},                       // 8
+}
+
+func TestMasterKeyKMS(t *testing.T) {
+	kms := NewKMS([32]byte{})
+	for i, test := range masterKeyKMSTests {
+		key, sealedKey, err := kms.GenerateKey(test.GenKeyID, test.GenContext)
+		if err != nil {
+			t.Errorf("Test %d: KMS failed to generate key: %v", i, err)
+		}
+		unsealedKey, err := kms.UnsealKey(test.UnsealKeyID, sealedKey, test.UnsealContext)
+		if err != nil && !test.ShouldFail {
+			t.Errorf("Test %d: KMS failed to unseal the generated key: %v", i, err)
+		}
+		if err == nil && test.ShouldFail {
+			t.Errorf("Test %d: KMS unsealed the generated successfully but should have failed", i)
+		}
+		if !test.ShouldFail && !bytes.Equal(key[:], unsealedKey[:]) {
+			t.Errorf("Test %d: The generated and unsealed key differ", i)
+		}
+	}
+}
+
+var contextWriteToTests = []struct {
+	Context      Context
+	ExpectedJSON string
+}{
+	{Context: Context{}, ExpectedJSON: "{}"},                                                    // 0
+	{Context: Context{"a": "b"}, ExpectedJSON: `{"a":"b"}`},                                     // 1
+	{Context: Context{"a": "b", "c": "d"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 2
+	{Context: Context{"c": "d", "a": "b"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 3
+	{Context: Context{"0": "1", "-": "2", ".": "#"}, ExpectedJSON: `{"-":"2",".":"#","0":"1"}`}, // 4
+}
+
+func TestContextWriteTo(t *testing.T) {
+	for i, test := range contextWriteToTests {
+		var jsonContext strings.Builder
+		if _, err := test.Context.WriteTo(&jsonContext); err != nil {
+			t.Errorf("Test %d: Failed to encode context: %v", i, err)
+			continue
+		}
+		if s := jsonContext.String(); s != test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
+		}
+	}
+}

cmd/crypto/metadata.go

@@ -0,0 +1,213 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// IsMultiPart returns true if the object metadata indicates
+// that it was uploaded using some form of server-side-encryption
+// and the S3 multipart API.
+func IsMultiPart(metadata map[string]string) bool {
+	if _, ok := metadata[SSEMultipart]; ok {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that it was uploaded using some form of server-side-encryption.
+//
+// IsEncrypted only checks whether the metadata contains at least
+// one entry indicating SSE-C or SSE-S3.
+func IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[SSEIV]; ok {
+		return true
+	}
+	if _, ok := metadata[SSESealAlgorithm]; ok {
+		return true
+	}
+	if IsMultiPart(metadata) {
+		return true
+	}
+	if S3.IsEncrypted(metadata) {
+		return true
+	}
+	if SSEC.IsEncrypted(metadata) {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-S3.
+func (s3) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[S3SealedKey]; ok {
+		return true
+	}
+	if _, ok := metadata[S3KMSKeyID]; ok {
+		return true
+	}
+	if _, ok := metadata[S3KMSSealedKey]; ok {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-C.
+func (ssec) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[SSECSealedKey]; ok {
+		return true
+	}
+	return false
+}
+
+// CreateMultipartMetadata adds the multipart flag entry to metadata
+// and returns modifed metadata. It allocates a new metadata map if
+// metadata is nil.
+func CreateMultipartMetadata(metadata map[string]string) map[string]string {
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+	metadata[SSEMultipart] = ""
+	return metadata
+}
+
+// CreateMetadata encodes the keyID, the sealed kms data key and the sealed key
+// into the metadata and returns the modified metadata. It allocates a new
+// metadata map if metadata is nil.
+func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+	}
+
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+	metadata[S3KMSKeyID] = keyID
+	metadata[SSESealAlgorithm] = sealedKey.Algorithm
+	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[S3SealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-S3 related values from the object metadata
+// and checks whether they are well-formed. It returns the KMS key-ID, the
+// sealed KMS key and the sealed object key on success.
+func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[SSEIV]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[SSESealAlgorithm]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[S3SealedKey]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-S3"}
+	}
+	keyID, ok = metadata[S3KMSKeyID]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"}
+	}
+	b64KMSSealedKey, ok := metadata[S3KMSSealedKey]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"}
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm {
+		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return keyID, kmsKey, sealedKey, Error{"The internal sealed key for SSE-S3 is invalid"}
+	}
+	kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+	if err != nil {
+		return keyID, kmsKey, sealedKey, Error{"The internal sealed KMS data key for SSE-S3 is invalid"}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return keyID, kmsKey, sealedKey, nil
+}
+
+// CreateMetadata encodes the sealed key into the metadata and returns the modified metadata.
+// It allocates a new metadata map if metadata is nil.
+func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
+	}
+
+	if metadata == nil {
+		metadata = map[string]string{}
+	}
+	metadata[SSESealAlgorithm] = SealAlgorithm
+	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[SSECSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-C related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success.
+func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[SSEIV]
+	if !ok {
+		return sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[SSESealAlgorithm]
+	if !ok {
+		return sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[SSECSealedKey]
+	if !ok {
+		return sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-C"}
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
+		return sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return sealedKey, Error{"The internal sealed key for SSE-C is invalid"}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return sealedKey, nil
+}

cmd/crypto/metadata_test.go

@@ -0,0 +1,361 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"encoding/base64"
+	"testing"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+var isMultipartTests = []struct {
+	Metadata  map[string]string
+	Multipart bool
+}{
+	{Multipart: true, Metadata: map[string]string{SSEMultipart: ""}},                           // 0
+	{Multipart: true, Metadata: map[string]string{"X-Minio-Internal-Encrypted-Multipart": ""}}, // 1
+	{Multipart: true, Metadata: map[string]string{SSEMultipart: "some-value"}},                 // 2
+	{Multipart: false, Metadata: map[string]string{"": ""}},                                    // 3
+	{Multipart: false, Metadata: map[string]string{"X-Minio-Internal-EncryptedMultipart": ""}}, // 4
+}
+
+func TestIsMultipart(t *testing.T) {
+	for i, test := range isMultipartTests {
+		if isMultipart := IsMultiPart(test.Metadata); isMultipart != test.Multipart {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isMultipart, test.Multipart)
+		}
+	}
+}
+
+var isEncryptedTests = []struct {
+	Metadata  map[string]string
+	Encrypted bool
+}{
+	{Encrypted: true, Metadata: map[string]string{SSEMultipart: ""}},                               // 0
+	{Encrypted: true, Metadata: map[string]string{SSEIV: ""}},                                      // 1
+	{Encrypted: true, Metadata: map[string]string{SSESealAlgorithm: ""}},                           // 2
+	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
+	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
+	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
+	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
+	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
+}
+
+func TestIsEncrypted(t *testing.T) {
+	for i, test := range isEncryptedTests {
+		if isEncrypted := IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
+		}
+	}
+}
+
+var s3IsEncryptedTests = []struct {
+	Metadata  map[string]string
+	Encrypted bool
+}{
+	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
+	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
+	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
+	{Encrypted: false, Metadata: map[string]string{SSECSealedKey: ""}},                             // 3
+	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
+	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
+	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
+	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
+}
+
+func TestS3IsEncrypted(t *testing.T) {
+	for i, test := range s3IsEncryptedTests {
+		if isEncrypted := S3.IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
+		}
+	}
+}
+
+var ssecIsEncryptedTests = []struct {
+	Metadata  map[string]string
+	Encrypted bool
+}{
+	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
+	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
+	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
+	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
+	{Encrypted: false, Metadata: map[string]string{S3SealedKey: ""}},                               // 4
+	{Encrypted: false, Metadata: map[string]string{S3KMSKeyID: ""}},                                // 5
+	{Encrypted: false, Metadata: map[string]string{S3KMSSealedKey: ""}},                            // 6
+	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
+	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
+}
+
+func TestSSECIsEncrypted(t *testing.T) {
+	for i, test := range ssecIsEncryptedTests {
+		if isEncrypted := SSEC.IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
+		}
+	}
+}
+
+var s3ParseMetadataTests = []struct {
+	Metadata    map[string]string
+	ExpectedErr error
+
+	DataKey   []byte
+	KeyID     string
+	SealedKey SealedKey
+}{
+	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}}, // 0
+	{
+		ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 1
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-S3"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 2
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 3
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
+		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 4
+	{
+		ExpectedErr: errInvalidInternalIV,
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: ""},
+		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 5
+	{
+		ExpectedErr: errInvalidInternalSealAlgorithm,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: "",
+		},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 6
+	{
+		ExpectedErr: Error{"The internal sealed key for SSE-S3 is invalid"},
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
+			S3KMSKeyID: "", S3KMSSealedKey: "",
+		},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+	}, // 7
+	{
+		ExpectedErr: Error{"The internal sealed KMS data key for SSE-S3 is invalid"},
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
+			S3KMSSealedKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
+		},
+		DataKey: []byte{}, KeyID: "key-1", SealedKey: SealedKey{},
+	}, // 8
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "", S3KMSSealedKey: "",
+		},
+		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{Algorithm: SealAlgorithm},
+	}, // 9
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: SealAlgorithm,
+			S3SealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), S3KMSKeyID: "key-1",
+			S3KMSSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
+		},
+		DataKey: make([]byte, 48), KeyID: "key-1", SealedKey: SealedKey{Algorithm: SealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
+	}, // 10
+}
+
+func TestS3ParseMetadata(t *testing.T) {
+	for i, test := range s3ParseMetadataTests {
+		keyID, dataKey, sealedKey, err := S3.ParseMetadata(test.Metadata)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if !bytes.Equal(dataKey, test.DataKey) {
+			t.Errorf("Test %d: got data key '%v' - want data key '%v'", i, dataKey, test.DataKey)
+		}
+		if keyID != test.KeyID {
+			t.Errorf("Test %d: got key-ID '%v' - want key-ID '%v'", i, keyID, test.KeyID)
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: got sealed key '%v' - want sealed key '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: got sealed key IV '%v' - want sealed key IV '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+	}
+}
+
+var ssecParseMetadataTests = []struct {
+	Metadata    map[string]string
+	ExpectedErr error
+
+	SealedKey SealedKey
+}{
+	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
+	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
+	{
+		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-C"},
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
+	}, // 2
+	{
+		ExpectedErr: errInvalidInternalIV,
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", SSECSealedKey: ""}, SealedKey: SealedKey{},
+	}, // 3
+	{
+		ExpectedErr: errInvalidInternalSealAlgorithm,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", SSECSealedKey: "",
+		},
+		SealedKey: SealedKey{},
+	}, // 4
+	{
+		ExpectedErr: Error{"The internal sealed key for SSE-C is invalid"},
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
+		},
+		SealedKey: SealedKey{},
+	}, // 5
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			SSECSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+		},
+		SealedKey: SealedKey{Algorithm: SealAlgorithm},
+	}, // 6
+	{
+		ExpectedErr: nil,
+		Metadata: map[string]string{
+			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: InsecureSealAlgorithm,
+			SSECSealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
+		},
+		SealedKey: SealedKey{Algorithm: InsecureSealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
+	}, // 7
+}
+
+func TestCreateMultipartMetadata(t *testing.T) {
+	metadata := CreateMultipartMetadata(nil)
+	if v, ok := metadata[SSEMultipart]; !ok || v != "" {
+		t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", SSEMultipart, v, "")
+	}
+}
+
+func TestSSECParseMetadata(t *testing.T) {
+	for i, test := range ssecParseMetadataTests {
+		sealedKey, err := SSEC.ParseMetadata(test.Metadata)
+		if err != test.ExpectedErr {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: got sealed key '%v' - want sealed key '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: got sealed key IV '%v' - want sealed key IV '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+	}
+}
+
+var s3CreateMetadataTests = []struct {
+	KeyID         string
+	SealedDataKey []byte
+	SealedKey     SealedKey
+}{
+	{KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}},
+}
+
+func TestS3CreateMetadata(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+	for i, test := range s3CreateMetadataTests {
+		metadata := S3.CreateMetadata(nil, test.KeyID, test.SealedDataKey, test.SealedKey)
+		keyID, kmsKey, sealedKey, err := S3.ParseMetadata(metadata)
+		if err != nil {
+			t.Errorf("Test %d: failed to parse metadata: %v", i, err)
+			continue
+		}
+		if keyID != test.KeyID {
+			t.Errorf("Test %d: Key-ID mismatch: got '%s' - want '%s'", i, keyID, test.KeyID)
+		}
+		if !bytes.Equal(kmsKey, test.SealedDataKey) {
+			t.Errorf("Test %d: sealed KMS data mismatch: got '%v' - want '%v'", i, kmsKey, test.SealedDataKey)
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: seal algorithm mismatch: got '%s' - want '%s'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: IV mismatch: got '%v' - want '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: sealed key mismatch: got '%v' - want '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+	}
+
+	defer func() {
+		if err := recover(); err == nil || err != logger.ErrCritical {
+			t.Errorf("Expected '%s' panic for invalid seal algorithm but got '%s'", logger.ErrCritical, err)
+		}
+	}()
+	_ = S3.CreateMetadata(nil, "", []byte{}, SealedKey{Algorithm: InsecureSealAlgorithm})
+}
+
+var ssecCreateMetadataTests = []SealedKey{
+	{Algorithm: SealAlgorithm},
+	{IV: [32]byte{0xff}, Key: [64]byte{0x7e}, Algorithm: SealAlgorithm},
+}
+
+func TestSSECCreateMetadata(t *testing.T) {
+	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
+	logger.Disable = true
+	for i, test := range s3CreateMetadataTests {
+		metadata := SSEC.CreateMetadata(nil, test.SealedKey)
+		sealedKey, err := SSEC.ParseMetadata(metadata)
+		if err != nil {
+			t.Errorf("Test %d: failed to parse metadata: %v", i, err)
+			continue
+		}
+		if sealedKey.Algorithm != test.SealedKey.Algorithm {
+			t.Errorf("Test %d: seal algorithm mismatch: got '%s' - want '%s'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
+		}
+		if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) {
+			t.Errorf("Test %d: IV mismatch: got '%v' - want '%v'", i, sealedKey.IV, test.SealedKey.IV)
+		}
+		if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) {
+			t.Errorf("Test %d: sealed key mismatch: got '%v' - want '%v'", i, sealedKey.Key, test.SealedKey.Key)
+		}
+	}
+
+	defer func() {
+		if err := recover(); err == nil || err != logger.ErrCritical {
+			t.Errorf("Expected '%s' panic for invalid seal algorithm but got '%s'", logger.ErrCritical, err)
+		}
+	}()
+	_ = SSEC.CreateMetadata(nil, SealedKey{Algorithm: InsecureSealAlgorithm})
+}

cmd/crypto/sse.go

@@ -25,16 +25,54 @@ import (
 )
 
 const (
+	// SSEMultipart is the metadata key indicating that the object
+	// was uploaded using the S3 multipart API and stored using
+	// some from of server-side-encryption.
+	SSEMultipart = "X-Minio-Internal-Encrypted-Multipart"
+
+	// SSEIV is the metadata key referencing the random initialization
+	// vector (IV) used for SSE-S3 and SSE-C key derivation.
+	SSEIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
+
+	// SSESealAlgorithm is the metadata key referencing the algorithm
+	// used by SSE-C and SSE-S3 to encrypt the object.
+	SSESealAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
+
+	// SSECSealedKey is the metadata key referencing the sealed object-key for SSE-C.
+	SSECSealedKey = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
+
 	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
 	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+
 	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
 	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
 	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+
 	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
 	// by KMS. It is only used for SSE-S3 + KMS.
 	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
 )
 
+const (
+	// SealAlgorithm is the encryption/sealing algorithm used to derive & seal
+	// the key-encryption-key and to en/decrypt the object data.
+	SealAlgorithm = "DAREv2-HMAC-SHA256"
+
+	// InsecureSealAlgorithm is the legacy encryption/sealing algorithm used
+	// to derive & seal the key-encryption-key and to en/decrypt the object data.
+	// This algorithm should not be used for new objects because its key derivation
+	// is not optimal. See: https://github.com/minio/minio/pull/6121
+	InsecureSealAlgorithm = "DARE-SHA256"
+)
+
+// String returns the SSE domain as string. For SSE-S3 the
+// domain is "SSE-S3".
+func (s3) String() string { return "SSE-S3" }
+
+// String returns the SSE domain as string. For SSE-C the
+// domain is "SSE-C".
+func (ssec) String() string { return "SSE-C" }
+
 // EncryptSinglePart encrypts an io.Reader which must be the
 // the body of a single-part PUT request.
 func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
@@ -45,6 +83,14 @@ func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
 	return r
 }
 
+// EncryptMultiPart encrypts an io.Reader which must be the body of
+// multi-part PUT request. It derives an unique encryption key from
+// the partID and the object key.
+func EncryptMultiPart(r io.Reader, partID int, key ObjectKey) io.Reader {
+	partKey := key.DerivePartKey(uint32(partID))
+	return EncryptSinglePart(r, ObjectKey(partKey))
+}
+
 // DecryptSinglePart decrypts an io.Writer which must an object
 // uploaded with the single-part PUT API. The offset and length
 // specify the requested range.

cmd/crypto/sse_test.go

@@ -0,0 +1,31 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import "testing"
+
+func TestS3String(t *testing.T) {
+	const Domain = "SSE-S3"
+	if domain := S3.String(); domain != Domain {
+		t.Errorf("S3's string method returns wrong domain: got '%s' - want '%s'", domain, Domain)
+	}
+}
+
+func TestSSECString(t *testing.T) {
+	const Domain = "SSE-C"
+	if domain := SSEC.String(); domain != Domain {
+		t.Errorf("SSEC's string method returns wrong domain: got '%s' - want '%s'", domain, Domain)
+	}
+}

cmd/disk-cache-config.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"path/filepath"
 	"strings"
 
@@ -44,6 +45,15 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 	if err = json.Unmarshal(data, _cfg); err != nil {
 		return err
 	}
+
+	if _cfg.Expiry < 0 {
+		return errors.New("config expiry value should not be negative")
+	}
+
+	if _cfg.MaxUse < 0 {
+		return errors.New("config max use value should not be null or negative")
+	}
+
 	if _, err = parseCacheDrives(_cfg.Drives); err != nil {
 		return err
 	}

cmd/disk-cache.go

@@ -32,14 +32,10 @@ import (
 	"github.com/djherbis/atime"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/wildcard"
-
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/wildcard"
 )
 
-// list of all errors that can be ignored in tree walk operation in disk cache
-var cacheTreeWalkIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errVolumeNotFound, errFileNotFound)
-
 const (
 	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
 	cacheSizeMultiplier  = 100
@@ -291,40 +287,32 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - list of fsObjects
-func listDirCacheFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks []*cacheFSObjects) listDirFunc {
-	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string, err error) {
+func listDirCacheFactory(isLeaf isLeafFunc, disks []*cacheFSObjects) listDirFunc {
+	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string) {
 		var entries []string
 		for _, disk := range disks {
 			// ignore disk-caches that might be missing/offline
 			if disk == nil {
 				continue
 			}
-			fs := disk.FSObjects
-			entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 
-			// For any reason disk was deleted or goes offline, continue
-			// and list from other disks if possible.
+			fs := disk.FSObjects
+			var err error
+			entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 			if err != nil {
-				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
-					continue
-				}
-				return nil, err
+				continue
 			}
 
 			// Filter entries that have the prefix prefixEntry.
 			entries = filterMatchingPrefix(entries, prefixEntry)
 			dirs = append(dirs, entries...)
 		}
-		return dirs, nil
+		return dirs
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
-		var cacheEntries []string
-		cacheEntries, err = listCacheDirs(bucket, prefixDir, prefixEntry)
-		if err != nil {
-			return nil, false, err
-		}
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
+		cacheEntries := listCacheDirs(bucket, prefixDir, prefixEntry)
 		for _, entry := range cacheEntries {
 			// Find elements in entries which are not in mergedEntries
 			idx := sort.SearchStrings(mergedEntries, entry)
@@ -335,7 +323,7 @@ func listDirCacheFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks [
 			mergedEntries = append(mergedEntries, entry)
 			sort.Strings(mergedEntries)
 		}
-		return mergedEntries, false, nil
+		return mergedEntries, false
 	}
 	return listDir
 }
@@ -371,7 +359,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			return fs.isObjectDir(bucket, object)
 		}
 
-		listDir := listDirCacheFactory(isLeaf, cacheTreeWalkIgnoredErrs, c.cache.cfs)
+		listDir := listDirCacheFactory(isLeaf, c.cache.cfs)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 

cmd/dummy-object-layer_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"context"
 	"io"
-	"time"
 
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
@@ -132,14 +131,6 @@ func (api *DummyObjectLayer) ListObjectsHeal(ctx context.Context, bucket, prefix
 	return
 }
 
-func (api *DummyObjectLayer) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) (info []VolumeLockInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ClearLocks(context.Context, []VolumeLockInfo) (err error) {
-	return
-}
-
 func (api *DummyObjectLayer) SetBucketPolicy(context.Context, string, *policy.Policy) (err error) {
 	return
 }

cmd/encryption-v1.go

@@ -640,15 +640,12 @@ func DecryptAllBlocksCopyRequest(client io.Writer, r *http.Request, bucket, obje
 // DecryptBlocksRequest - setup a struct which can decrypt many concatenated encrypted data
 // parts information helps to know the boundaries of each encrypted data block.
 func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object string, startOffset, length int64, objInfo ObjectInfo, copySource bool) (io.WriteCloser, int64, int64, error) {
-	seqNumber, encStartOffset, encLength := getEncryptedStartOffset(startOffset, length)
-
-	// Encryption length cannot be bigger than the file size, if it is
-	// which is allowed in AWS S3, we simply default to EncryptedSize().
-	if encLength+encStartOffset > objInfo.EncryptedSize() {
-		encLength = objInfo.EncryptedSize() - encStartOffset
-	}
+	var seqNumber uint32
+	var encStartOffset, encLength int64
 
 	if len(objInfo.Parts) == 0 || !objInfo.IsEncryptedMultipart() {
+		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
+
 		var writer io.WriteCloser
 		var err error
 		if copySource {
@@ -662,6 +659,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 		return writer, encStartOffset, encLength, nil
 	}
 
+	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
 	var partStartIndex int
 	var partStartOffset = startOffset
 	// Skip parts until final offset maps to a particular part offset.
@@ -716,15 +714,62 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 		w.customerKeyHeader = r.Header.Get(SSECopyCustomerKey)
 	}
 
-	if err := w.buildDecrypter(partStartIndex + 1); err != nil {
+	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
 		return nil, 0, 0, err
 	}
 
 	return w, encStartOffset, encLength, nil
 }
 
-// getEncryptedStartOffset - fetch sequence number, encrypted start offset and encrypted length.
-func getEncryptedStartOffset(offset, length int64) (seqNumber uint32, encOffset int64, encLength int64) {
+// getEncryptedMultipartsOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
+func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (uint32, int64, int64) {
+
+	// Calculate encrypted offset of a multipart object
+	computeEncOffset := func(off int64, obj ObjectInfo) (seqNumber uint32, encryptedOffset int64, err error) {
+		var curPartEndOffset uint64
+		var prevPartsEncSize int64
+		for _, p := range obj.Parts {
+			size, decErr := sio.DecryptedSize(uint64(p.Size))
+			if decErr != nil {
+				err = errObjectTampered // assign correct error type
+				return
+			}
+			if off < int64(curPartEndOffset+size) {
+				seqNumber, encryptedOffset, _ = getEncryptedSinglePartOffsetLength(off-int64(curPartEndOffset), 1, obj)
+				encryptedOffset += int64(prevPartsEncSize)
+				break
+			}
+			curPartEndOffset += size
+			prevPartsEncSize += p.Size
+		}
+		return
+	}
+
+	// Calculate the encrypted start offset corresponding to the plain offset
+	seqNumber, encStartOffset, _ := computeEncOffset(offset, obj)
+	// Calculate also the encrypted end offset corresponding to plain offset + plain length
+	_, encEndOffset, _ := computeEncOffset(offset+length-1, obj)
+
+	// encLength is the diff between encrypted end offset and encrypted start offset + one package size
+	// to ensure all encrypted data are covered
+	encLength := encEndOffset - encStartOffset + (64*1024 + 32)
+
+	// Calculate total size of all parts
+	var totalPartsLength int64
+	for _, p := range obj.Parts {
+		totalPartsLength += p.Size
+	}
+
+	// Set encLength to maximum possible value if it exceeded total parts size
+	if encLength+encStartOffset > totalPartsLength {
+		encLength = totalPartsLength - encStartOffset
+	}
+
+	return seqNumber, encStartOffset, encLength
+}
+
+// getEncryptedSinglePartOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
+func getEncryptedSinglePartOffsetLength(offset, length int64, objInfo ObjectInfo) (seqNumber uint32, encOffset int64, encLength int64) {
 	onePkgSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
 
 	seqNumber = uint32(offset / sseDAREPackageBlockSize)
@@ -742,6 +787,9 @@ func getEncryptedStartOffset(offset, length int64) (seqNumber uint32, encOffset
 		encLength += onePkgSize
 	}
 
+	if encLength+encOffset > objInfo.EncryptedSize() {
+		encLength = objInfo.EncryptedSize() - encOffset
+	}
 	return seqNumber, encOffset, encLength
 }
 
@@ -786,11 +834,23 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if !o.IsEncrypted() {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
-	size, err := sio.DecryptedSize(uint64(o.Size))
-	if err != nil {
-		err = errObjectTampered // assign correct error type
+	if len(o.Parts) == 0 || !o.IsEncryptedMultipart() {
+		size, err := sio.DecryptedSize(uint64(o.Size))
+		if err != nil {
+			err = errObjectTampered // assign correct error type
+		}
+		return int64(size), err
 	}
-	return int64(size), err
+
+	var size int64
+	for _, part := range o.Parts {
+		partSize, err := sio.DecryptedSize(uint64(part.Size))
+		if err != nil {
+			return 0, errObjectTampered
+		}
+		size += int64(partSize)
+	}
+	return size, nil
 }
 
 // EncryptedSize returns the size of the object after encryption.

cmd/format-fs.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -55,24 +56,22 @@ type formatFSVersionDetect struct {
 	} `json:"fs"`
 }
 
+// Generic structure to manage both v1 and v2 structures
+type formatFS struct {
+	formatMetaV1
+	FS interface{} `json:"fs"`
+}
+
 // Returns the latest "fs" format V1
 func newFormatFSV1() (format *formatFSV1) {
 	f := &formatFSV1{}
 	f.Version = formatMetaVersionV1
 	f.Format = formatBackendFS
+	f.ID = mustGetUUID()
 	f.FS.Version = formatFSVersionV1
 	return f
 }
 
-// Returns the latest "fs" format V2
-func newFormatFSV2() (format *formatFSV2) {
-	f := &formatFSV2{}
-	f.Version = formatMetaVersionV1
-	f.Format = formatBackendFS
-	f.FS.Version = formatFSVersionV2
-	return f
-}
-
 // Returns the field formatMetaV1.Format i.e the string "fs" which is never likely to change.
 // We do not use this function in XL to get the format as the file is not fcntl-locked on XL.
 func formatMetaGetFormatBackendFS(r io.ReadSeeker) (string, error) {
@@ -115,7 +114,16 @@ func formatFSMigrateV1ToV2(ctx context.Context, wlk *lock.LockedFile, fsPath str
 		return err
 	}
 
-	return jsonSave(wlk.File, newFormatFSV2())
+	formatV1 := formatFSV1{}
+	if err = jsonLoad(wlk, &formatV1); err != nil {
+		return err
+	}
+
+	formatV2 := formatFSV2{}
+	formatV2.formatMetaV1 = formatV1.formatMetaV1
+	formatV2.FS.Version = formatFSVersionV2
+
+	return jsonSave(wlk.File, formatV2)
 }
 
 // Migrate the "fs" backend.
@@ -179,6 +187,12 @@ func createFormatFS(ctx context.Context, fsFormatPath string) error {
 // migrate the backend when we are actively working on the backend.
 func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, err error) {
 	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
+
+	// Add a deployment ID, if it does not exist.
+	if err := formatFSFixDeploymentID(fsFormatPath); err != nil {
+		return nil, err
+	}
+
 	// Any read on format.json should be done with read-lock.
 	// Any write on format.json should be done with write-lock.
 	for {
@@ -235,7 +249,8 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			rlk.Close()
 			// Hold write lock during migration so that we do not disturb any
 			// minio processes running in parallel.
-			wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+			var wlk *lock.LockedFile
+			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
 			if err == lock.ErrAlreadyLocked {
 				// Lock already present, sleep and attempt again.
 				time.Sleep(100 * time.Millisecond)
@@ -253,7 +268,94 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			// Successfully migrated, now try to hold a read-lock on format.json
 			continue
 		}
-
+		var id string
+		if id, err = formatFSGetDeploymentID(rlk); err != nil {
+			rlk.Close()
+			return nil, err
+		}
+		logger.SetDeploymentID(id)
 		return rlk, nil
 	}
 }
+
+func formatFSGetDeploymentID(rlk *lock.RLockedFile) (id string, err error) {
+	format := &formatFS{}
+	if err := jsonLoad(rlk, format); err != nil {
+		return "", err
+	}
+	return format.ID, nil
+}
+
+// Generate a deployment ID if one does not exist already.
+func formatFSFixDeploymentID(fsFormatPath string) error {
+	rlk, err := lock.RLockedOpenFile(fsFormatPath)
+	if err == nil {
+		// format.json can be empty in a rare condition when another
+		// minio process just created the file but could not hold lock
+		// and write to it.
+		var fi os.FileInfo
+		fi, err = rlk.Stat()
+		if err != nil {
+			rlk.Close()
+			return err
+		}
+		if fi.Size() == 0 {
+			rlk.Close()
+			return nil
+		}
+	}
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	formatBackend, err := formatMetaGetFormatBackendFS(rlk)
+	if err != nil {
+		rlk.Close()
+		return err
+	}
+	if formatBackend != formatBackendFS {
+		rlk.Close()
+		return fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)
+	}
+
+	format := &formatFS{}
+	err = jsonLoad(rlk, format)
+	rlk.Close()
+	if err != nil {
+		return err
+	}
+
+	// Check if it needs to be updated
+	if format.ID != "" {
+		return nil
+	}
+
+	for {
+		wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+		if err == lock.ErrAlreadyLocked {
+			// Lock already present, sleep and attempt again.
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		if err != nil {
+			return err
+		}
+		defer wlk.Close()
+
+		err = jsonLoad(wlk, format)
+		if err != nil {
+			return err
+		}
+
+		// Check if it needs to be updated
+		if format.ID != "" {
+			return nil
+		}
+		format.ID = mustGetUUID()
+		return jsonSave(wlk, format)
+	}
+
+}

cmd/format-meta.go

@@ -49,4 +49,6 @@ type formatMetaV1 struct {
 	Version string `json:"version"`
 	// Format indicates the backend format type, supports two values 'xl' and 'fs'.
 	Format string `json:"format"`
+	// ID is the identifier for the minio deployment
+	ID string `json:"id"`
 }

cmd/format-xl.go

@@ -22,11 +22,13 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"sync"
 
 	"encoding/hex"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/logger"
 	sha256 "github.com/minio/sha256-simd"
 )
 
@@ -86,10 +88,11 @@ type formatXLV1 struct {
 // Represents the V2 backend disk structure version
 // under `.minio.sys` and actual data namespace.
 // formatXLV2 - structure holds format config version '2'.
+// The V2 format to support "large bucket" support where a bucket
+// can span multiple erasure sets.
 type formatXLV2 struct {
-	Version string `json:"version"`
-	Format  string `json:"format"`
-	XL      struct {
+	formatMetaV1
+	XL struct {
 		Version string `json:"version"` // Version of 'xl' format.
 		This    string `json:"this"`    // This field carries assigned disk uuid.
 		// Sets field carries the input disk order generated the first
@@ -107,9 +110,8 @@ type formatXLV2 struct {
 // In .minio.sys/multipart we have:
 // sha256(bucket/object)/uploadID/[xl.json, part.1, part.2 ....]
 type formatXLV3 struct {
-	Version string `json:"version"`
-	Format  string `json:"format"`
-	XL      struct {
+	formatMetaV1
+	XL struct {
 		Version string `json:"version"` // Version of 'xl' format.
 		This    string `json:"this"`    // This field carries assigned disk uuid.
 		// Sets field carries the input disk order generated the first
@@ -127,6 +129,7 @@ func newFormatXLV3(numSets int, setLen int) *formatXLV3 {
 	format := &formatXLV3{}
 	format.Version = formatMetaVersionV1
 	format.Format = formatBackendXL
+	format.ID = mustGetUUID()
 	format.XL.Version = formatXLVersionV3
 	format.XL.DistributionAlgo = formatXLVersionV2DistributionAlgo
 	format.XL.Sets = make([][]string, numSets)
@@ -443,6 +446,112 @@ func checkFormatXLValues(formats []*formatXLV3) error {
 	return nil
 }
 
+// Get Deployment ID for the XL sets from format.json.
+// This need not be in quorum. Even if one of the format.json
+// file has this value, we assume it is valid.
+// If more than one format.json's have different id, it is considered a corrupt
+// backend format.
+func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (string, error) {
+	var deploymentID string
+	for _, format := range formats {
+		if format == nil || format.ID == "" {
+			continue
+		}
+		if reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) {
+			// Found an ID in one of the format.json file
+			// Set deploymentID for the first time.
+			if deploymentID == "" {
+				deploymentID = format.ID
+			} else if deploymentID != format.ID {
+				// DeploymentID found earlier doesn't match with the
+				// current format.json's ID.
+				return "", errCorruptedFormat
+			}
+		}
+	}
+	return deploymentID, nil
+}
+
+// formatXLFixDeploymentID - Add deployment id if it is not present.
+func formatXLFixDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
+	// Acquire lock on format.json
+	mutex := newNSLock(globalIsDistXL)
+	formatLock := mutex.NewNSLock(minioMetaBucket, formatConfigFile)
+	if err = formatLock.GetLock(globalHealingTimeout); err != nil {
+		return err
+	}
+	defer formatLock.Unlock()
+
+	// Attempt to load all `format.json` from all disks.
+	var sErrs []error
+	formats, sErrs := loadFormatXLAll(storageDisks)
+	for i, sErr := range sErrs {
+		if _, ok := formatCriticalErrors[sErr]; ok {
+			return fmt.Errorf("Disk %s: %s", globalEndpoints[i], sErr)
+		}
+	}
+
+	for index := range formats {
+		// If the XL sets do not match, set those formats to nil,
+		// We do not have to update the ID on those format.json file.
+		if formats[index] != nil && !reflect.DeepEqual(formats[index].XL.Sets, refFormat.XL.Sets) {
+			formats[index] = nil
+		}
+	}
+	refFormat.ID, err = formatXLGetDeploymentID(refFormat, formats)
+	if err != nil {
+		return err
+	}
+
+	// If ID is set, then some other node got the lock
+	// before this node could and generated an ID
+	// for the deployment. No need to generate one.
+	if refFormat.ID != "" {
+		return nil
+	}
+
+	// ID is generated for the first time,
+	// We set the ID in all the formats and update.
+	refFormat.ID = mustGetUUID()
+	for _, format := range formats {
+		if format != nil {
+			format.ID = refFormat.ID
+		}
+	}
+	// Deployment ID needs to be set on all the disks.
+	// Save `format.json` across all disks.
+	return saveFormatXLAll(ctx, storageDisks, formats)
+
+}
+
+// Update only the valid local disks which have not been updated before.
+func formatXLFixLocalDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) error {
+	// If this server was down when the deploymentID was updated
+	// then we make sure that we update the local disks with the deploymentID.
+	for index, storageDisk := range storageDisks {
+		if globalEndpoints[index].IsLocal && storageDisk != nil && storageDisk.IsOnline() {
+			format, err := loadFormatXL(storageDisk)
+			if err != nil {
+				// Disk can be offline etc.
+				// ignore the errors seen here.
+				continue
+			}
+			if format.ID != "" {
+				continue
+			}
+			if !reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) {
+				continue
+			}
+			format.ID = refFormat.ID
+			if err := saveFormatXL(storageDisk, format); err != nil {
+				logger.LogIf(ctx, err)
+				return fmt.Errorf("Unable to save format.json, %s", err)
+			}
+		}
+	}
+	return nil
+}
+
 // Get backend XL format in quorum `format.json`.
 func getFormatXLInQuorum(formats []*formatXLV3) (*formatXLV3, error) {
 	formatHashes := make([]string, len(formats))

cmd/format-xl_test.go

@@ -345,8 +345,10 @@ func TestCheckFormatXLValue(t *testing.T) {
 		// Invalid XL format version "2".
 		{
 			&formatXLV3{
-				Version: "2",
-				Format:  "XL",
+				formatMetaV1: formatMetaV1{
+					Version: "2",
+					Format:  "XL",
+				},
 				XL: struct {
 					Version          string     `json:"version"`
 					This             string     `json:"this"`
@@ -361,8 +363,10 @@ func TestCheckFormatXLValue(t *testing.T) {
 		// Invalid XL format "Unknown".
 		{
 			&formatXLV3{
-				Version: "1",
-				Format:  "Unknown",
+				formatMetaV1: formatMetaV1{
+					Version: "1",
+					Format:  "Unknown",
+				},
 				XL: struct {
 					Version          string     `json:"version"`
 					This             string     `json:"this"`
@@ -377,8 +381,10 @@ func TestCheckFormatXLValue(t *testing.T) {
 		// Invalid XL format version "0".
 		{
 			&formatXLV3{
-				Version: "1",
-				Format:  "XL",
+				formatMetaV1: formatMetaV1{
+					Version: "1",
+					Format:  "XL",
+				},
 				XL: struct {
 					Version          string     `json:"version"`
 					This             string     `json:"this"`
@@ -466,6 +472,61 @@ func TestGetFormatXLInQuorumCheck(t *testing.T) {
 	}
 }
 
+// Tests formatXLGetDeploymentID()
+func TestGetXLID(t *testing.T) {
+	setCount := 2
+	disksPerSet := 8
+
+	format := newFormatXLV3(setCount, disksPerSet)
+	formats := make([]*formatXLV3, 16)
+
+	for i := 0; i < setCount; i++ {
+		for j := 0; j < disksPerSet; j++ {
+			newFormat := *format
+			newFormat.XL.This = format.XL.Sets[i][j]
+			formats[i*disksPerSet+j] = &newFormat
+		}
+	}
+
+	// Return a format from list of formats in quorum.
+	quorumFormat, err := getFormatXLInQuorum(formats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check if the reference format and input formats are same.
+	var id string
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
+		t.Fatal(err)
+	}
+
+	if id == "" {
+		t.Fatal("ID cannot be empty.")
+	}
+
+	formats[0] = nil
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
+		t.Fatal(err)
+	}
+	if id == "" {
+		t.Fatal("ID cannot be empty.")
+	}
+
+	formats[1].XL.Sets[0][0] = "bad-uuid"
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
+		t.Fatal(err)
+	}
+
+	if id == "" {
+		t.Fatal("ID cannot be empty.")
+	}
+
+	formats[2].ID = "bad-id"
+	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat {
+		t.Fatal("Unexpected Success")
+	}
+}
+
 // Initialize new format sets.
 func TestNewFormatSets(t *testing.T) {
 	setCount := 2

cmd/fs-v1-multipart.go

@@ -543,25 +543,11 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 		// All parts except the last part has to be atleast 5MB.
 		if !isMinAllowedPartSize(fi.Size()) {
-			err = PartTooSmall{
+			return oi, PartTooSmall{
 				PartNumber: part.PartNumber,
 				PartSize:   fi.Size(),
 				PartETag:   part.ETag,
 			}
-			logger.LogIf(ctx, err)
-			return oi, err
-		}
-
-		// TODO: Make necessary changes in future as explained in the below comment.
-		// All parts except the last part has to be of same size. We are introducing this
-		// check to see if any clients break. If clients do not break then we can optimize
-		// multipart PutObjectPart by writing the part at the right offset using pwrite()
-		// so that we don't need to do background append at all. i.e by the time we get
-		// CompleteMultipartUpload we already have the full file available which can be
-		// renamed to the main name-space.
-		if partSize != fi.Size() {
-			logger.LogIf(ctx, PartsSizeUnequal{})
-			return oi, PartsSizeUnequal{}
 		}
 	}
 

cmd/fs-v1.go

@@ -151,16 +151,6 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	// or cause changes on backend format.
 	fs.fsFormatRlk = rlk
 
-	// Initialize notification system.
-	if err = globalNotificationSys.Init(fs); err != nil {
-		return nil, uiErrUnableToReadFromBackend(err).Msg("Unable to initialize notification system")
-	}
-
-	// Initialize policy system.
-	if err = globalPolicySys.Init(fs); err != nil {
-		return nil, uiErrUnableToReadFromBackend(err).Msg("Unable to initialize policy system")
-	}
-
 	if !fs.diskMount {
 		go fs.diskUsage(globalServiceDoneCh)
 	}
@@ -275,18 +265,6 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
 	return storageInfo
 }
 
-// Locking operations
-
-// ListLocks - List namespace locks held in object layer
-func (fs *FSObjects) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	return []VolumeLockInfo{}, NotImplemented{}
-}
-
-// ClearLocks - Clear namespace locks held in object layer
-func (fs *FSObjects) ClearLocks(ctx context.Context, info []VolumeLockInfo) error {
-	return NotImplemented{}
-}
-
 /// Bucket operations
 
 // getBucketDir - will convert incoming bucket names to
@@ -978,13 +956,15 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
 // is a leaf or non-leaf entry.
 func (fs *FSObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool, err error) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
+		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil {
-			return nil, false, err
+			logger.LogIf(context.Background(), err)
+			return
 		}
 		entries, delayIsLeaf = filterListEntries(bucket, prefixDir, entries, prefixEntry, isLeaf)
-		return entries, delayIsLeaf, nil
+		return entries, delayIsLeaf
 	}
 
 	// Return list factory instance.

cmd/gateway-main.go

@@ -159,6 +159,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Initialize gateway config.
 	initConfig()
 
+	// Load logger subsystem
+	loadLoggers()
+
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
@@ -170,8 +173,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	initNSLock(false) // Enable local namespace lock.
 
 	// Create new notification system.
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, EndpointList{})
-	logger.FatalIf(err, "Unable to create new notification system")
+	globalNotificationSys = NewNotificationSys(globalServerConfig, EndpointList{})
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()

cmd/gateway-unsupported.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
@@ -137,20 +136,6 @@ func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, sr
 	return objInfo, NotImplemented{}
 }
 
-// Locking operations
-
-// ListLocks lists namespace locks held in object layer
-func (a GatewayUnsupported) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return []VolumeLockInfo{}, NotImplemented{}
-}
-
-// ClearLocks clears namespace locks held in object layer
-func (a GatewayUnsupported) ClearLocks(ctx context.Context, info []VolumeLockInfo) error {
-	logger.LogIf(ctx, NotImplemented{})
-	return NotImplemented{}
-}
-
 // RefreshBucketPolicy refreshes cache policy with what's on disk.
 func (a GatewayUnsupported) RefreshBucketPolicy(ctx context.Context, bucket string) error {
 	logger.LogIf(ctx, NotImplemented{})

cmd/gateway/gcs/gateway-gcs.go

@@ -554,16 +554,16 @@ func isGCSMarker(marker string) bool {
 
 // ListObjects - lists all blobs in GCS bucket filtered by prefix
 func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (minio.ListObjectsInfo, error) {
+	if maxKeys == 0 {
+		return minio.ListObjectsInfo{}, nil
+	}
+
 	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
 		Delimiter: delimiter,
 		Prefix:    prefix,
 		Versions:  false,
 	})
 
-	isTruncated := false
-	nextMarker := ""
-	prefixes := []string{}
-
 	// To accommodate S3-compatible applications using
 	// ListObjectsV1 to use object keys as markers to control the
 	// listing of objects, we use the following encoding scheme to
@@ -574,83 +574,86 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
 	//   prefixing "{minio}" to the GCS continuation token,
 	//   e.g, "{minio}CgRvYmoz"
 	//
-	// - Application supplied markers are used as-is to list
-	//   object keys that appear after it in the lexicographical order.
+	// - Application supplied markers are transformed to a
+	//   GCS continuation token.
 
 	// If application is using GCS continuation token we should
 	// strip the gcsTokenPrefix we added.
-	gcsMarker := isGCSMarker(marker)
-	if gcsMarker {
-		it.PageInfo().Token = strings.TrimPrefix(marker, gcsTokenPrefix)
+	token := ""
+	if marker != "" {
+		if isGCSMarker(marker) {
+			token = strings.TrimPrefix(marker, gcsTokenPrefix)
+		} else {
+			token = toGCSPageToken(marker)
+		}
 	}
+	nextMarker := ""
 
-	it.PageInfo().MaxSize = maxKeys
+	var prefixes []string
+	var objects []minio.ObjectInfo
+	var nextPageToken string
+	var err error
 
-	objects := []minio.ObjectInfo{}
+	pager := iterator.NewPager(it, maxKeys, token)
 	for {
-		if len(objects) >= maxKeys {
-			// check if there is one next object and
-			// if that one next object is our hidden
-			// metadata folder, then just break
-			// otherwise we've truncated the output
-			attrs, _ := it.Next()
-			if attrs != nil && attrs.Prefix == minio.GatewayMinioSysTmp {
-				break
-			}
-
-			isTruncated = true
-			break
-		}
-
-		attrs, err := it.Next()
-		if err == iterator.Done {
-			break
-		}
+		gcsObjects := make([]*storage.ObjectAttrs, 0)
+		nextPageToken, err = pager.NextPage(&gcsObjects)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return minio.ListObjectsInfo{}, gcsToObjectError(err, bucket, prefix)
 		}
 
-		nextMarker = toGCSPageToken(attrs.Name)
+		for _, attrs := range gcsObjects {
 
-		if attrs.Prefix == minio.GatewayMinioSysTmp {
-			// We don't return our metadata prefix.
-			continue
-		}
-		if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
-			// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
-			// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
-			// which will be helpful to observe the "directory structure" for debugging purposes.
-			if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
-				strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
+			// Due to minio.GatewayMinioSysTmp keys being skipped, the number of objects + prefixes
+			// returned may not total maxKeys. This behavior is compatible with the S3 spec which
+			// allows the response to include less keys than maxKeys.
+			if attrs.Prefix == minio.GatewayMinioSysTmp {
+				// We don't return our metadata prefix.
 				continue
 			}
-		}
-		if attrs.Prefix != "" {
-			prefixes = append(prefixes, attrs.Prefix)
-			continue
-		}
-		if !gcsMarker && attrs.Name <= marker {
-			// if user supplied a marker don't append
-			// objects until we reach marker (and skip it).
-			continue
+			if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
+				// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
+				// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
+				// which will be helpful to observe the "directory structure" for debugging purposes.
+				if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
+					strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
+					continue
+				}
+			}
+
+			if attrs.Prefix != "" {
+				prefixes = append(prefixes, attrs.Prefix)
+			} else {
+				objects = append(objects, fromGCSAttrsToObjectInfo(attrs))
+			}
+
+			// The NextMarker property should only be set in the response if a delimiter is used
+			if delimiter != "" {
+				if attrs.Prefix > nextMarker {
+					nextMarker = attrs.Prefix
+				} else if attrs.Name > nextMarker {
+					nextMarker = attrs.Name
+				}
+			}
 		}
 
-		objects = append(objects, minio.ObjectInfo{
-			Name:            attrs.Name,
-			Bucket:          attrs.Bucket,
-			ModTime:         attrs.Updated,
-			Size:            attrs.Size,
-			ETag:            minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C)),
-			UserDefined:     attrs.Metadata,
-			ContentType:     attrs.ContentType,
-			ContentEncoding: attrs.ContentEncoding,
-		})
+		// Exit the loop if at least one item can be returned from
+		// the current page or there are no more pages available
+		if nextPageToken == "" || len(prefixes)+len(objects) > 0 {
+			break
+		}
+	}
+
+	if nextPageToken == "" {
+		nextMarker = ""
+	} else if nextMarker != "" {
+		nextMarker = gcsTokenPrefix + toGCSPageToken(nextMarker)
 	}
 
 	return minio.ListObjectsInfo{
-		IsTruncated: isTruncated,
-		NextMarker:  gcsTokenPrefix + nextMarker,
+		IsTruncated: nextPageToken != "",
+		NextMarker:  nextMarker,
 		Prefixes:    prefixes,
 		Objects:     objects,
 	}, nil
@@ -658,8 +661,8 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
 
 // ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix
 func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
-	if continuationToken == "" && startAfter != "" {
-		continuationToken = toGCSPageToken(startAfter)
+	if maxKeys == 0 {
+		return minio.ListObjectsV2Info{ContinuationToken: continuationToken}, nil
 	}
 
 	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
@@ -668,62 +671,62 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
 		Versions:  false,
 	})
 
-	isTruncated := false
-	it.PageInfo().MaxSize = maxKeys
-
-	if continuationToken != "" {
-		// If client sends continuationToken, set it
-		it.PageInfo().Token = continuationToken
-	} else {
-		// else set the continuationToken to return
-		continuationToken = it.PageInfo().Token
-		if continuationToken != "" {
-			// If GCS SDK sets continuationToken, it means there are more than maxKeys in the current page
-			// and the response will be truncated
-			isTruncated = true
-		}
+	token := continuationToken
+	if token == "" && startAfter != "" {
+		token = toGCSPageToken(startAfter)
 	}
 
 	var prefixes []string
 	var objects []minio.ObjectInfo
+	var nextPageToken string
+	var err error
 
+	pager := iterator.NewPager(it, maxKeys, token)
 	for {
-		attrs, err := it.Next()
-		if err == iterator.Done {
-			break
-		}
-
+		gcsObjects := make([]*storage.ObjectAttrs, 0)
+		nextPageToken, err = pager.NextPage(&gcsObjects)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return minio.ListObjectsV2Info{}, gcsToObjectError(err, bucket, prefix)
 		}
 
-		if attrs.Prefix == minio.GatewayMinioSysTmp {
-			// We don't return our metadata prefix.
-			continue
-		}
-		if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
-			// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
-			// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
-			// which will be helpful to observe the "directory structure" for debugging purposes.
-			if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
-				strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
+		for _, attrs := range gcsObjects {
+
+			// Due to minio.GatewayMinioSysTmp keys being skipped, the number of objects + prefixes
+			// returned may not total maxKeys. This behavior is compatible with the S3 spec which
+			// allows the response to include less keys than maxKeys.
+			if attrs.Prefix == minio.GatewayMinioSysTmp {
+				// We don't return our metadata prefix.
 				continue
 			}
+			if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
+				// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
+				// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
+				// which will be helpful to observe the "directory structure" for debugging purposes.
+				if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
+					strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
+					continue
+				}
+			}
+
+			if attrs.Prefix != "" {
+				prefixes = append(prefixes, attrs.Prefix)
+			} else {
+				objects = append(objects, fromGCSAttrsToObjectInfo(attrs))
+			}
 		}
 
-		if attrs.Prefix != "" {
-			prefixes = append(prefixes, attrs.Prefix)
-			continue
+		// Exit the loop if at least one item can be returned from
+		// the current page or there are no more pages available
+		if nextPageToken == "" || len(prefixes)+len(objects) > 0 {
+			break
 		}
-
-		objects = append(objects, fromGCSAttrsToObjectInfo(attrs))
 	}
 
 	return minio.ListObjectsV2Info{
-		IsTruncated:           isTruncated,
+		IsTruncated:           nextPageToken != "",
 		ContinuationToken:     continuationToken,
-		NextContinuationToken: continuationToken,
+		NextContinuationToken: nextPageToken,
 		Prefixes:              prefixes,
 		Objects:               objects,
 	}, nil

cmd/gateway/s3/gateway-s3.go

@@ -20,7 +20,9 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"math/rand"
 	"strings"
+	"time"
 
 	"github.com/minio/cli"
 	miniogo "github.com/minio/minio-go"
@@ -121,6 +123,31 @@ func (g *S3) Name() string {
 	return s3Backend
 }
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
+const (
+	letterIdxBits = 6                    // 6 bits to represent a letter index
+	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
+	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
+)
+
+// randString generates random names and prepends them with a known prefix.
+func randString(n int, src rand.Source, prefix string) string {
+	b := make([]byte, n)
+	// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
+	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
+		if remain == 0 {
+			cache, remain = src.Int63(), letterIdxMax
+		}
+		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
+			b[i] = letterBytes[idx]
+			i--
+		}
+		cache >>= letterIdxBits
+		remain--
+	}
+	return prefix + string(b[0:30-len(prefix)])
+}
+
 // newS3 - Initializes a new client by auto probing S3 server signature.
 func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 	if url == "" {
@@ -137,13 +164,13 @@ func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+	if _, err = clnt.BucketExists(probeBucketName); err != nil {
 		clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
 		if err != nil {
 			return nil, err
 		}
-		if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+		if _, err = clnt.BucketExists(probeBucketName); err != nil {
 			return nil, err
 		}
 	}

cmd/generic-handlers.go

@@ -27,7 +27,6 @@ import (
 
 	"github.com/minio/minio-go/pkg/set"
 
-	etcd "github.com/coreos/etcd/client"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
@@ -653,7 +652,7 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 	}
 	sr, err := globalDNSConfig.Get(bucket)
 	if err != nil {
-		if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
+		if err == dns.ErrNoEntriesFound {
 			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
 		} else {
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -724,6 +723,26 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	l.handler.ServeHTTP(w, r)
 }
 
+// requestIDHeaderHandler sets x-amz-request-id header.
+// Previously, this value was set right before a response
+// was sent to the client.So, logger and Error response XML
+// were not using this value.
+// This is set here so that this header can be logged as
+// part of the log entry and Error response XML.
+type requestIDHeaderHandler struct {
+	handler http.Handler
+}
+
+func addrequestIDHeader(h http.Handler) http.Handler {
+	return requestIDHeaderHandler{handler: h}
+}
+
+func (s requestIDHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Set unique request ID for each response.
+	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
+	s.handler.ServeHTTP(w, r)
+}
+
 type securityHeaderHandler struct {
 	handler http.Handler
 }

cmd/globals.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/minio/minio-go/pkg/set"
 
-	etcd "github.com/coreos/etcd/client"
+	etcd "github.com/coreos/etcd/clientv3"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/fatih/color"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -204,7 +204,7 @@ var (
 	globalRPCAPIVersion = RPCVersion{3, 0, 0}
 
 	// Allocated etcd endpoint for config and bucket DNS.
-	globalEtcdClient etcd.Client
+	globalEtcdClient *etcd.Client
 
 	// Allocated DNS config wrapper over etcd client.
 	globalDNSConfig dns.Config

cmd/handler-utils.go

@@ -17,8 +17,10 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"io"
+	"io/ioutil"
 	"mime/multipart"
 	"net"
 	"net/http"
@@ -114,40 +116,54 @@ var userMetadataKeyPrefixes = []string{
 	"X-Minio-Meta-",
 }
 
-// extractMetadataFromHeader extracts metadata from HTTP header.
-func extractMetadataFromHeader(ctx context.Context, header http.Header) (map[string]string, error) {
-	if header == nil {
-		logger.LogIf(ctx, errInvalidArgument)
-		return nil, errInvalidArgument
+// extractMetadata extracts metadata from HTTP header and HTTP queryString.
+func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]string, err error) {
+	query := r.URL.Query()
+	header := r.Header
+	metadata = make(map[string]string)
+	// Extract all query values.
+	err = extractMetadataFromMap(ctx, query, metadata)
+	if err != nil {
+		return nil, err
 	}
-	metadata := make(map[string]string)
 
+	// Extract all header values.
+	err = extractMetadataFromMap(ctx, header, metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	// Success.
+	return metadata, nil
+}
+
+// extractMetadata extracts metadata from map values.
+func extractMetadataFromMap(ctx context.Context, v map[string][]string, m map[string]string) error {
+	if v == nil {
+		logger.LogIf(ctx, errInvalidArgument)
+		return errInvalidArgument
+	}
 	// Save all supported headers.
 	for _, supportedHeader := range supportedHeaders {
-		canonicalHeader := http.CanonicalHeaderKey(supportedHeader)
-		// HTTP headers are case insensitive, look for both canonical
-		// and non canonical entries.
-		if _, ok := header[canonicalHeader]; ok {
-			metadata[supportedHeader] = header.Get(canonicalHeader)
-		} else if _, ok := header[supportedHeader]; ok {
-			metadata[supportedHeader] = header.Get(supportedHeader)
+		if value, ok := v[http.CanonicalHeaderKey(supportedHeader)]; ok {
+			m[supportedHeader] = value[0]
+		} else if value, ok := v[supportedHeader]; ok {
+			m[supportedHeader] = value[0]
 		}
 	}
-
-	// Go through all other headers for any additional headers that needs to be saved.
-	for key := range header {
-		if key != http.CanonicalHeaderKey(key) {
-			logger.LogIf(ctx, errInvalidArgument)
-			return nil, errInvalidArgument
-		}
+	for key := range v {
 		for _, prefix := range userMetadataKeyPrefixes {
-			if strings.HasPrefix(key, prefix) {
-				metadata[key] = header.Get(key)
+			if !strings.HasPrefix(strings.ToLower(key), strings.ToLower(prefix)) {
+				continue
+			}
+			value, ok := v[key]
+			if ok {
+				m[key] = strings.Join(value, ",")
 				break
 			}
 		}
 	}
-	return metadata, nil
+	return nil
 }
 
 // The Query string for the redirect URL the client is
@@ -221,6 +237,19 @@ func extractPostPolicyFormValues(ctx context.Context, form *multipart.Form) (fil
 		return nil, "", 0, nil, err
 	}
 
+	// this means that filename="" was not specified for file key and Go has
+	// an ugly way of handling this situation. Refer here
+	// https://golang.org/src/mime/multipart/formdata.go#L61
+	if len(form.File) == 0 {
+		var b = &bytes.Buffer{}
+		for _, v := range formValues["File"] {
+			b.WriteString(v)
+		}
+		fileSize = int64(b.Len())
+		filePart = ioutil.NopCloser(b)
+		return filePart, fileName, fileSize, formValues, nil
+	}
+
 	// Iterator until we find a valid File field and break
 	for k, v := range form.File {
 		canonicalFormName := http.CanonicalHeaderKey(k)
@@ -255,7 +284,6 @@ func extractPostPolicyFormValues(ctx context.Context, form *multipart.Form) (fil
 			break
 		}
 	}
-
 	return filePart, fileName, fileSize, formValues, nil
 }
 

cmd/handler-utils_test.go

@@ -165,9 +165,9 @@ func TestExtractMetadataHeaders(t *testing.T) {
 				"x-amz-meta-appid": []string{"amz-meta"},
 			},
 			metadata: map[string]string{
-				"X-Amz-Meta-Appid": "amz-meta",
+				"x-amz-meta-appid": "amz-meta",
 			},
-			shouldFail: true,
+			shouldFail: false,
 		},
 		// Empty header input returns empty metadata.
 		{
@@ -179,7 +179,8 @@ func TestExtractMetadataHeaders(t *testing.T) {
 
 	// Validate if the extracting headers.
 	for i, testCase := range testCases {
-		metadata, err := extractMetadataFromHeader(context.Background(), testCase.header)
+		metadata := make(map[string]string)
+		err := extractMetadataFromMap(context.Background(), testCase.header, metadata)
 		if err != nil && !testCase.shouldFail {
 			t.Fatalf("Test %d failed to extract metadata: %v", i+1, err)
 		}

cmd/healthcheck-handler.go

@@ -17,10 +17,12 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"net/http"
+	"os"
 	"runtime"
+
+	"github.com/minio/minio/cmd/logger"
 )
 
 const (
@@ -40,19 +42,51 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
-// LivenessCheckHandler -- checks if server can ListBuckets internally. If not, server is
-// considered to have failed and needs to be restarted.
+// LivenessCheckHandler -- checks if server can reach its disks internally.
+// If not, server is considered to have failed and needs to be restarted.
 // Liveness probes are used to detect situations where application (minio)
 // has gone into a state where it can not recover except by being restarted.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "LivenessCheckHandler")
+
 	objLayer := newObjectLayerFn()
 	// Service not initialized yet
 	if objLayer == nil {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
-	// List buckets is unsuccessful, means server is having issues, send 503 service unavailable
-	if _, err := objLayer.ListBuckets(context.Background()); err != nil {
+
+	s := objLayer.StorageInfo(ctx)
+	// Gateways don't provide disk info
+	if s.Backend.Type == Unknown {
+		// ListBuckets to confirm gateway backend is up
+		if _, err := objLayer.ListBuckets(ctx); err != nil {
+			writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+			return
+		}
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
+	var totalLocalDisks int
+	var erroredDisks int
+	for _, endpoint := range globalEndpoints {
+		// Check only if local disks are accessible, we do not have
+		// to reach to rest of the other servers in a distributed setup.
+		if endpoint.IsLocal {
+			totalLocalDisks++
+			// Attempt a stat to backend, any error resulting
+			// from this Stat() operation is considered as backend
+			// is not available, count them as errors.
+			if _, err := os.Stat(endpoint.Path); err != nil {
+				logger.LogIf(ctx, err)
+				erroredDisks++
+			}
+		}
+	}
+	// If all exported local disks have errored, we simply let kubernetes
+	// take us down.
+	if totalLocalDisks == erroredDisks {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}

cmd/local-admin-client.go

@@ -23,7 +23,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"time"
 
 	"github.com/minio/minio/cmd/logger"
 )
@@ -51,16 +50,6 @@ func (lc localAdminClient) ReInitFormat(dryRun bool) error {
 	return objectAPI.ReloadFormat(context.Background(), dryRun)
 }
 
-// ListLocks - Fetches lock information from local lock instrumentation.
-func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return nil, errServerNotInitialized
-	}
-
-	return objectAPI.ListLocks(context.Background(), bucket, prefix, duration)
-}
-
 // ServerInfo - Returns the server info of this server.
 func (lc localAdminClient) ServerInfo() (sid ServerInfoData, e error) {
 	if globalBootTime.IsZero() {

cmd/local-admin-client_test.go

@@ -17,10 +17,7 @@
 package cmd
 
 import (
-	"fmt"
-	"os"
 	"testing"
-	"time"
 )
 
 func TestLocalAdminClientSignalService(t *testing.T) {
@@ -31,10 +28,6 @@ func TestLocalAdminClientReInitFormat(t *testing.T) {
 	testAdminCmdRunnerReInitFormat(t, &localAdminClient{})
 }
 
-func TestLocalAdminClientListLocks(t *testing.T) {
-	testAdminCmdRunnerListLocks(t, &localAdminClient{})
-}
-
 func TestLocalAdminClientServerInfo(t *testing.T) {
 	testAdminCmdRunnerServerInfo(t, &localAdminClient{})
 }
@@ -50,91 +43,3 @@ func TestLocalAdminClientWriteTmpConfig(t *testing.T) {
 func TestLocalAdminClientCommitConfig(t *testing.T) {
 	testAdminCmdRunnerCommitConfig(t, &localAdminClient{})
 }
-
-func TestListLocksInfo(t *testing.T) {
-	// reset global variables to start afresh.
-	resetTestGlobals()
-	// Initialize minio server config.
-	rootPath, err := newTestConfig(globalMinioDefaultRegion)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	// Initializing new XL objectLayer.
-	objAPI, _, xlErr := initTestXLObjLayer()
-	if xlErr != nil {
-		t.Fatalf("failed to init object layer")
-	}
-	// Make objLayer available to all internal services via globalObjectAPI.
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objAPI
-	globalObjLayerMutex.Unlock()
-	// Set globalIsXL to indicate that the setup uses an erasure code backend.
-	// initialize NSLock.
-	isDistXL := false
-	initNSLock(isDistXL)
-
-	var nsMutex *nsLockMap
-
-	nsMutex = objAPI.(*xlSets).sets[0].nsMutex
-
-	// Acquire a few locks to populate lock instrumentation.
-	// Take 10 read locks on bucket1/prefix1/obj1
-	for i := 0; i < 10; i++ {
-		readLk := nsMutex.NewNSLock("bucket1", "prefix1/obj1")
-		if readLk.GetRLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
-			t.Errorf("Failed to get read lock on iteration %d", i)
-		}
-	}
-
-	// Take write locks on bucket1/prefix/obj{11..19}
-	for i := 0; i < 10; i++ {
-		wrLk := nsMutex.NewNSLock("bucket1", fmt.Sprintf("prefix1/obj%d", 10+i))
-		if wrLk.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
-			t.Errorf("Failed to get write lock on iteration %d", i)
-		}
-	}
-
-	client := &localAdminClient{}
-
-	testCases := []struct {
-		bucket   string
-		prefix   string
-		duration time.Duration
-		numLocks int
-	}{
-		// Test 1 - Matches all the locks acquired above.
-		{
-			bucket:   "bucket1",
-			prefix:   "prefix1",
-			duration: time.Duration(0 * time.Second),
-			numLocks: 20,
-		},
-		// Test 2 - Bucket doesn't match.
-		{
-			bucket:   "bucket",
-			prefix:   "prefix1",
-			duration: time.Duration(0 * time.Second),
-			numLocks: 0,
-		},
-		// Test 3 - Prefix doesn't match.
-		{
-			bucket:   "bucket1",
-			prefix:   "prefix11",
-			duration: time.Duration(0 * time.Second),
-			numLocks: 0,
-		},
-	}
-
-	for i, test := range testCases {
-		actual, err := client.ListLocks(test.bucket, test.prefix, test.duration)
-		if err != nil {
-			t.Fatalf("unexpected error %v", err)
-		}
-		if len(actual) != test.numLocks {
-			t.Errorf("Test %d - Expected %d locks but observed %d locks",
-				i+1, test.numLocks, len(actual))
-		}
-	}
-}

cmd/lock-instrument.go

@@ -1,276 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-)
-
-type statusType string
-
-const (
-	runningStatus statusType = "Running"
-	blockedStatus statusType = "Blocked"
-)
-
-type lockType string
-
-const (
-	debugRLockStr lockType = "RLock"
-	debugWLockStr lockType = "WLock"
-)
-
-// debugLockInfo - represents a single lock's information, uniquely identified by opsID.
-// See debugLockInfoPerVolumePath for more context.
-type debugLockInfo struct {
-	// "RLock" or "WLock".
-	lType lockType
-	// Contains the backtrace of incl. the function which called (r)(un)lock.
-	lockSource string
-	// Status can be running/blocked.
-	status statusType
-	// Time of last status update.
-	since time.Time
-}
-
-// debugLockInfoPerVolumePath - lock state information on all locks held on (volume, path).
-type debugLockInfoPerVolumePath struct {
-	counters *lockStat                // Holds stats of lock held on (volume, path)
-	lockInfo map[string]debugLockInfo // Lock information per operation ID.
-}
-
-// LockInfoOriginMismatch - represents error when lock origin don't match.
-type LockInfoOriginMismatch struct {
-	volume     string
-	path       string
-	opsID      string
-	lockSource string
-}
-
-func (l LockInfoOriginMismatch) Error() string {
-	return fmt.Sprintf("No lock state stored for the lock originated at \"%s\", for <volume> %s, <path> %s, <opsID> %s",
-		l.lockSource, l.volume, l.path, l.opsID)
-}
-
-// LockInfoVolPathMissing - represents error when lock information is missing for a given (volume, path).
-type LockInfoVolPathMissing struct {
-	volume string
-	path   string
-}
-
-func (l LockInfoVolPathMissing) Error() string {
-	return fmt.Sprintf("No entry in debug Lock Map for Volume: %s, path: %s", l.volume, l.path)
-}
-
-// LockInfoOpsIDNotFound - represents error when lock info entry for a given operation ID doesn't exist.
-type LockInfoOpsIDNotFound struct {
-	volume string
-	path   string
-	opsID  string
-}
-
-func (l LockInfoOpsIDNotFound) Error() string {
-	return fmt.Sprintf("No entry in lock info for <Operation ID> %s, <volume> %s, <path> %s", l.opsID, l.volume, l.path)
-}
-
-// LockInfoStateNotBlocked - represents error when lock info isn't in blocked state when it should be.
-type LockInfoStateNotBlocked struct {
-	volume string
-	path   string
-	opsID  string
-}
-
-func (l LockInfoStateNotBlocked) Error() string {
-	return fmt.Sprintf("Lock state should be \"Blocked\" for <volume> %s, <path> %s, <opsID> %s", l.volume, l.path, l.opsID)
-}
-
-// Initialize lock info for given (volume, path).
-func (n *nsLockMap) initLockInfoForVolumePath(param nsParam) {
-	n.debugLockMap[param] = &debugLockInfoPerVolumePath{
-		lockInfo: make(map[string]debugLockInfo),
-		counters: &lockStat{},
-	}
-}
-
-// Change the state of the lock from Blocked to Running.
-func (n *nsLockMap) statusBlockedToRunning(param nsParam, lockSource, opsID string, readLock bool) error {
-	// This function is called outside nsLockMap.mutex.Lock(), so must be held explicitly.
-	ctx := context.Background()
-	n.lockMapMutex.Lock()
-	defer n.lockMapMutex.Unlock()
-
-	// Check whether the lock info entry for <volume, path> pair already exists.
-	_, ok := n.debugLockMap[param]
-	if !ok {
-		logger.LogIf(ctx, LockInfoVolPathMissing{param.volume, param.path})
-		return LockInfoVolPathMissing{param.volume, param.path}
-	}
-
-	// Check whether lock info entry for the given `opsID` exists.
-	lockInfo, ok := n.debugLockMap[param].lockInfo[opsID]
-	if !ok {
-		logger.LogIf(ctx, LockInfoOpsIDNotFound{param.volume, param.path, opsID})
-		return LockInfoOpsIDNotFound{param.volume, param.path, opsID}
-	}
-
-	// Check whether lockSource is same.
-	if lockInfo.lockSource != lockSource {
-		logger.LogIf(ctx, LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource})
-		return LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource}
-	}
-
-	// Status of the lock should be set to "Blocked".
-	if lockInfo.status != blockedStatus {
-		logger.LogIf(ctx, LockInfoStateNotBlocked{param.volume, param.path, opsID})
-		return LockInfoStateNotBlocked{param.volume, param.path, opsID}
-	}
-	// Change lock status to running and update the time.
-	n.debugLockMap[param].lockInfo[opsID] = newDebugLockInfo(lockSource, runningStatus, readLock)
-
-	// Update global lock stats.
-	n.counters.lockGranted()
-	// Update (volume, pair) lock stats.
-	n.debugLockMap[param].counters.lockGranted()
-	return nil
-}
-
-// newDebugLockInfo - Constructs a debugLockInfo value given lock source, status and type.
-func newDebugLockInfo(lockSource string, status statusType, readLock bool) debugLockInfo {
-	var lType lockType
-	if readLock {
-		lType = debugRLockStr
-	} else {
-		lType = debugWLockStr
-	}
-	return debugLockInfo{
-		lockSource: lockSource,
-		lType:      lType,
-		status:     status,
-		since:      UTCNow(),
-	}
-}
-
-// Change the state of the lock to Blocked.
-func (n *nsLockMap) statusNoneToBlocked(param nsParam, lockSource, opsID string, readLock bool) error {
-	_, ok := n.debugLockMap[param]
-	if !ok {
-		// Lock info entry for (volume, pair) doesn't exist, initialize it.
-		n.initLockInfoForVolumePath(param)
-	}
-
-	// Mark lock status blocked for given opsID.
-	n.debugLockMap[param].lockInfo[opsID] = newDebugLockInfo(lockSource, blockedStatus, readLock)
-	// Update global lock stats.
-	n.counters.lockWaiting()
-	// Update (volume, path) lock stats.
-	n.debugLockMap[param].counters.lockWaiting()
-	return nil
-}
-
-// Change the state of the lock from Blocked to none.
-func (n *nsLockMap) statusBlockedToNone(param nsParam, lockSource, opsID string, readLock bool) error {
-	_, ok := n.debugLockMap[param]
-	ctx := context.Background()
-	if !ok {
-		logger.LogIf(ctx, LockInfoVolPathMissing{param.volume, param.path})
-		return LockInfoVolPathMissing{param.volume, param.path}
-	}
-
-	// Check whether lock info entry for the given `opsID` exists.
-	lockInfo, ok := n.debugLockMap[param].lockInfo[opsID]
-	if !ok {
-		logger.LogIf(ctx, LockInfoOpsIDNotFound{param.volume, param.path, opsID})
-		return LockInfoOpsIDNotFound{param.volume, param.path, opsID}
-	}
-
-	// Check whether lockSource is same.
-	if lockInfo.lockSource != lockSource {
-		logger.LogIf(ctx, LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource})
-		return LockInfoOriginMismatch{param.volume, param.path, opsID, lockSource}
-	}
-
-	// Status of the lock should be set to "Blocked".
-	if lockInfo.status != blockedStatus {
-		logger.LogIf(ctx, LockInfoStateNotBlocked{param.volume, param.path, opsID})
-		return LockInfoStateNotBlocked{param.volume, param.path, opsID}
-	}
-
-	// Update global lock stats.
-	n.counters.lockTimedOut()
-	// Update (volume, path) lock stats.
-	n.debugLockMap[param].counters.lockTimedOut()
-	return nil
-}
-
-// deleteLockInfoEntry - Deletes the lock information for given (volume, path).
-// Called when nsLk.ref count is 0.
-func (n *nsLockMap) deleteLockInfoEntryForVolumePath(param nsParam) error {
-	// delete the lock info for the given operation.
-	if _, found := n.debugLockMap[param]; !found {
-		logger.LogIf(context.Background(), LockInfoVolPathMissing{param.volume, param.path})
-		return LockInfoVolPathMissing{param.volume, param.path}
-	}
-
-	// The following stats update is relevant only in case of a
-	// ForceUnlock. In case of the last unlock on a (volume,
-	// path), this would be a no-op.
-	volumePathLocks := n.debugLockMap[param]
-	for _, lockInfo := range volumePathLocks.lockInfo {
-		granted := lockInfo.status == runningStatus
-		// Update global and (volume, path) stats.
-		n.counters.lockRemoved(granted)
-		volumePathLocks.counters.lockRemoved(granted)
-	}
-	delete(n.debugLockMap, param)
-	return nil
-}
-
-// deleteLockInfoEntry - Deletes lock info entry for given opsID.
-// Called when the nsLk ref count for the given (volume, path) is
-// not 0.
-func (n *nsLockMap) deleteLockInfoEntryForOps(param nsParam, opsID string) error {
-	ctx := context.Background()
-	// delete the lock info for the given operation.
-	infoMap, found := n.debugLockMap[param]
-	if !found {
-		logger.LogIf(ctx, LockInfoVolPathMissing{param.volume, param.path})
-		return LockInfoVolPathMissing{param.volume, param.path}
-	}
-	// The operation finished holding the lock on the resource, remove
-	// the entry for the given operation with the operation ID.
-	opsIDLock, foundInfo := infoMap.lockInfo[opsID]
-	if !foundInfo {
-		// Unlock request with invalid operation ID not accepted.
-		logger.LogIf(ctx, LockInfoOpsIDNotFound{param.volume, param.path, opsID})
-		return LockInfoOpsIDNotFound{param.volume, param.path, opsID}
-	}
-	// Update global and (volume, path) lock status.
-	granted := opsIDLock.status == runningStatus
-	n.counters.lockRemoved(granted)
-	infoMap.counters.lockRemoved(granted)
-	delete(infoMap.lockInfo, opsID)
-	return nil
-}
-
-// Return randomly generated string ID
-func getOpsID() string {
-	return mustGetUUID()
-}

cmd/lock-instrument_test.go

@@ -1,667 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"testing"
-)
-
-type lockStateCase struct {
-	volume      string
-	path        string
-	lockSource  string
-	opsID       string
-	readLock    bool // lock type.
-	setBlocked  bool // initialize the initial state to blocked.
-	expectedErr error
-	// Expected global lock stats.
-	expectedLockStatus statusType // Status of the lock Blocked/Running.
-
-	expectedGlobalLockCount  int // Total number of locks held across the system, includes blocked + held locks.
-	expectedBlockedLockCount int // Total blocked lock across the system.
-	expectedRunningLockCount int // Total successfully held locks (non-blocking).
-	// Expected lock status for given <volume, path> pair.
-	expectedVolPathLockCount    int // Total locks held for given <volume,path> pair, includes blocked locks.
-	expectedVolPathRunningCount int // Total succcesfully held locks for given <volume, path> pair.
-	expectedVolPathBlockCount   int // Total locks blocked on the given <volume, path> pair.
-}
-
-// Read entire state of the locks in the system and return.
-func getSystemLockState() (SystemLockState, error) {
-	globalNSMutex.lockMapMutex.Lock()
-	defer globalNSMutex.lockMapMutex.Unlock()
-
-	lockState := SystemLockState{}
-
-	lockState.TotalBlockedLocks = globalNSMutex.counters.blocked
-	lockState.TotalLocks = globalNSMutex.counters.total
-	lockState.TotalAcquiredLocks = globalNSMutex.counters.granted
-
-	for param, debugLock := range globalNSMutex.debugLockMap {
-		volLockInfo := VolumeLockInfo{}
-		volLockInfo.Bucket = param.volume
-		volLockInfo.Object = param.path
-		volLockInfo.LocksOnObject = debugLock.counters.total
-		volLockInfo.TotalBlockedLocks = debugLock.counters.blocked
-		volLockInfo.LocksAcquiredOnObject = debugLock.counters.granted
-		for opsID, lockInfo := range debugLock.lockInfo {
-			volLockInfo.LockDetailsOnObject = append(volLockInfo.LockDetailsOnObject, OpsLockState{
-				OperationID: opsID,
-				LockSource:  lockInfo.lockSource,
-				LockType:    lockInfo.lType,
-				Status:      lockInfo.status,
-				Since:       lockInfo.since,
-			})
-		}
-		lockState.LocksInfoPerObject = append(lockState.LocksInfoPerObject, volLockInfo)
-	}
-	return lockState, nil
-}
-
-// Asserts the lock counter from the global globalNSMutex inmemory lock with the expected one.
-func verifyGlobalLockStats(l lockStateCase, t *testing.T, testNum int) {
-	globalNSMutex.lockMapMutex.Lock()
-
-	// Verifying the lock stats.
-	if globalNSMutex.counters.total != int64(l.expectedGlobalLockCount) {
-		t.Errorf("Test %d: Expected the global lock counter to be %v, but got %v", testNum, int64(l.expectedGlobalLockCount),
-			globalNSMutex.counters.total)
-	}
-	// verify the count for total blocked locks.
-	if globalNSMutex.counters.blocked != int64(l.expectedBlockedLockCount) {
-		t.Errorf("Test %d: Expected the total blocked lock counter to be %v, but got %v", testNum, int64(l.expectedBlockedLockCount),
-			globalNSMutex.counters.blocked)
-	}
-	// verify the count for total running locks.
-	if globalNSMutex.counters.granted != int64(l.expectedRunningLockCount) {
-		t.Errorf("Test %d: Expected the total running lock counter to be %v, but got %v", testNum, int64(l.expectedRunningLockCount),
-			globalNSMutex.counters.granted)
-	}
-	globalNSMutex.lockMapMutex.Unlock()
-	// Verifying again with the JSON response of the lock info.
-	// Verifying the lock stats.
-	sysLockState, err := getSystemLockState()
-	if err != nil {
-		t.Fatalf("Obtaining lock info failed with <ERROR> %s", err)
-
-	}
-	if sysLockState.TotalLocks != int64(l.expectedGlobalLockCount) {
-		t.Errorf("Test %d: Expected the global lock counter to be %v, but got %v", testNum, int64(l.expectedGlobalLockCount),
-			sysLockState.TotalLocks)
-	}
-	// verify the count for total blocked locks.
-	if sysLockState.TotalBlockedLocks != int64(l.expectedBlockedLockCount) {
-		t.Errorf("Test %d: Expected the total blocked lock counter to be %v, but got %v", testNum, int64(l.expectedBlockedLockCount),
-			sysLockState.TotalBlockedLocks)
-	}
-	// verify the count for total running locks.
-	if sysLockState.TotalAcquiredLocks != int64(l.expectedRunningLockCount) {
-		t.Errorf("Test %d: Expected the total running lock counter to be %v, but got %v", testNum, int64(l.expectedRunningLockCount),
-			sysLockState.TotalAcquiredLocks)
-	}
-}
-
-// Verify the lock counter for entries of given <volume, path> pair.
-func verifyLockStats(l lockStateCase, t *testing.T, testNum int) {
-	globalNSMutex.lockMapMutex.Lock()
-	defer globalNSMutex.lockMapMutex.Unlock()
-	param := nsParam{l.volume, l.path}
-
-	// Verify the total locks (blocked+running) for given <vol,path> pair.
-	if globalNSMutex.debugLockMap[param].counters.total != int64(l.expectedVolPathLockCount) {
-		t.Errorf("Test %d: Expected the total lock count for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum,
-			param.volume, param.path, int64(l.expectedVolPathLockCount), globalNSMutex.debugLockMap[param].counters.total)
-	}
-	// Verify the total running locks for given <volume, path> pair.
-	if globalNSMutex.debugLockMap[param].counters.granted != int64(l.expectedVolPathRunningCount) {
-		t.Errorf("Test %d: Expected the total running locks for volume: \"%s\", path: \"%s\" to be %v, but got %v", testNum, param.volume, param.path,
-			int64(l.expectedVolPathRunningCount), globalNSMutex.debugLockMap[param].counters.granted)
-	}
-	// Verify the total blocked locks for givne <volume, path> pair.
-	if globalNSMutex.debugLockMap[param].counters.blocked != int64(l.expectedVolPathBlockCount) {
-		t.Errorf("Test %d:  Expected the total blocked locks for volume: \"%s\", path: \"%s\"  to be %v, but got %v", testNum, param.volume, param.path,
-			int64(l.expectedVolPathBlockCount), globalNSMutex.debugLockMap[param].counters.blocked)
-	}
-}
-
-// verifyLockState - function which asserts the expected lock info in the system with the actual values in the globalNSMutex.
-func verifyLockState(l lockStateCase, t *testing.T, testNum int) {
-	param := nsParam{l.volume, l.path}
-
-	verifyGlobalLockStats(l, t, testNum)
-	globalNSMutex.lockMapMutex.Lock()
-	// Verifying the lock statuS fields.
-	if debugLockMap, ok := globalNSMutex.debugLockMap[param]; ok {
-		if lockInfo, ok := debugLockMap.lockInfo[l.opsID]; ok {
-			// Validating the lock type filed in the debug lock information.
-			if l.readLock {
-				if lockInfo.lType != debugRLockStr {
-					t.Errorf("Test case %d: Expected the lock type in the lock debug info to be \"%s\"", testNum, debugRLockStr)
-				}
-			} else {
-				if lockInfo.lType != debugWLockStr {
-					t.Errorf("Test case %d: Expected the lock type in the lock debug info to be \"%s\"", testNum, debugWLockStr)
-				}
-			}
-
-			// // validating the lock origin.
-			// if l.lockSource != lockInfo.lockSource {
-			// 	t.Fatalf("Test %d: Expected the lock origin info to be \"%s\", but got \"%s\"", testNum, l.lockSource, lockInfo.lockSource)
-			// }
-			// validating the status of the lock.
-			if lockInfo.status != l.expectedLockStatus {
-				t.Errorf("Test %d: Expected the status of the lock to be \"%s\", but got \"%s\"", testNum, l.expectedLockStatus, lockInfo.status)
-			}
-		} else {
-			// Stop the tests if lock debug entry for given <volume, path> pair is not found.
-			t.Errorf("Test case %d: Expected an debug lock entry for opsID \"%s\"", testNum, l.opsID)
-		}
-	} else {
-		// To change the status the entry for given <volume, path> should exist in the lock info struct.
-		t.Errorf("Test case %d: Debug lock entry for volume: %s, path: %s doesn't exist", testNum, param.volume, param.path)
-	}
-	// verifyLockStats holds its own lock.
-	globalNSMutex.lockMapMutex.Unlock()
-
-	// verify the lock count.
-	verifyLockStats(l, t, testNum)
-}
-
-// TestNewDebugLockInfoPerVolumePath -  Validates the values initialized by newDebugLockInfoPerVolumePath().
-func TestNewDebugLockInfoPerVolumePath(t *testing.T) {
-	lockInfo := &debugLockInfoPerVolumePath{
-		lockInfo: make(map[string]debugLockInfo),
-		counters: &lockStat{},
-	}
-
-	if lockInfo.counters.total != 0 {
-		t.Errorf("Expected initial reference value of total locks to be 0, got %d", lockInfo.counters.total)
-	}
-	if lockInfo.counters.blocked != 0 {
-		t.Errorf("Expected initial reference of blocked locks to be 0, got %d", lockInfo.counters.blocked)
-	}
-	if lockInfo.counters.granted != 0 {
-		t.Errorf("Expected initial reference value of held locks to be 0, got %d", lockInfo.counters.granted)
-	}
-}
-
-// TestNsLockMapStatusBlockedToRunning - Validates the function for changing the lock state from blocked to running.
-func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
-
-	testCases := []struct {
-		volume      string
-		path        string
-		lockSource  string
-		opsID       string
-		readLock    bool // Read lock type.
-		setBlocked  bool // Initialize the initial state to blocked.
-		expectedErr error
-	}{
-		// Test case - 1.
-		{
-			volume:     "my-bucket",
-			path:       "my-object",
-			lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-			opsID:      "abcd1234",
-			readLock:   true,
-			setBlocked: true,
-			// expected metrics.
-			expectedErr: nil,
-		},
-		// Test case - 2.
-		// No entry for <volume, path> pair.
-		// So an attempt to change the state of the lock from `Blocked`->`Running` should fail.
-		{
-			volume:     "my-bucket",
-			path:       "my-object-2",
-			lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-			opsID:      "abcd1234",
-			readLock:   false,
-			setBlocked: false,
-			// expected metrics.
-			expectedErr: LockInfoVolPathMissing{"my-bucket", "my-object-2"},
-		},
-		// Test case - 3.
-		// Entry for the given operationID doesn't exist in the lock state info.
-		{
-			volume:     "my-bucket",
-			path:       "my-object",
-			lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-			opsID:      "ops-Id-not-registered",
-			readLock:   true,
-			setBlocked: false,
-			// expected metrics.
-			expectedErr: LockInfoOpsIDNotFound{"my-bucket", "my-object", "ops-Id-not-registered"},
-		},
-		// Test case - 4.
-		// Test case with non-existent lock origin.
-		{
-			volume:     "my-bucket",
-			path:       "my-object",
-			lockSource: "Bad Origin",
-			opsID:      "abcd1234",
-			readLock:   true,
-			setBlocked: false,
-			// expected metrics.
-			expectedErr: LockInfoOriginMismatch{"my-bucket", "my-object", "abcd1234", "Bad Origin"},
-		},
-		// Test case - 5.
-		// Test case with write lock.
-		{
-			volume:     "my-bucket",
-			path:       "my-object",
-			lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-			opsID:      "abcd1234",
-			readLock:   false,
-			setBlocked: true,
-			// expected metrics.
-			expectedErr: nil,
-		},
-	}
-
-	param := nsParam{testCases[0].volume, testCases[0].path}
-	// Testing before the initialization done.
-	// Since the data structures for
-	actualErr := globalNSMutex.statusBlockedToRunning(param, testCases[0].lockSource,
-		testCases[0].opsID, testCases[0].readLock)
-
-	expectedErr := LockInfoVolPathMissing{testCases[0].volume, testCases[0].path}
-	if actualErr != expectedErr {
-		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedErr, actualErr)
-	}
-
-	globalNSMutex = &nsLockMap{
-		// entries of <volume,path> -> stateInfo of locks, for instrumentation purpose.
-		debugLockMap: make(map[nsParam]*debugLockInfoPerVolumePath),
-		lockMap:      make(map[nsParam]*nsLock),
-	}
-
-	// Setting the lock info the be `nil`.
-	globalNSMutex.debugLockMap[param] = &debugLockInfoPerVolumePath{
-		lockInfo: nil, // setting the lockinfo to nil.
-		counters: &lockStat{},
-	}
-
-	actualErr = globalNSMutex.statusBlockedToRunning(param, testCases[0].lockSource,
-		testCases[0].opsID, testCases[0].readLock)
-
-	expectedOpsErr := LockInfoOpsIDNotFound{testCases[0].volume, testCases[0].path, testCases[0].opsID}
-	if actualErr != expectedOpsErr {
-		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedOpsErr, actualErr)
-	}
-
-	// Next case: ase whether an attempt to change the state of the lock to "Running" done,
-	// but the initial state if already "Running". Such an attempt should fail
-	globalNSMutex.debugLockMap[param] = &debugLockInfoPerVolumePath{
-		lockInfo: make(map[string]debugLockInfo),
-		counters: &lockStat{},
-	}
-
-	// Setting the status of the lock to be "Running".
-	// The initial state of the lock should set to "Blocked", otherwise its not possible to change the state from "Blocked" -> "Running".
-	globalNSMutex.debugLockMap[param].lockInfo[testCases[0].opsID] = debugLockInfo{
-		lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-		status:     "Running", // State set to "Running". Should fail with `LockInfoStateNotBlocked`.
-		since:      UTCNow(),
-	}
-
-	actualErr = globalNSMutex.statusBlockedToRunning(param, testCases[0].lockSource,
-		testCases[0].opsID, testCases[0].readLock)
-
-	expectedBlockErr := LockInfoStateNotBlocked{testCases[0].volume, testCases[0].path, testCases[0].opsID}
-	if actualErr != expectedBlockErr {
-		t.Fatalf("Errors mismatch: Expected: \"%s\", got: \"%s\"", expectedBlockErr, actualErr)
-	}
-
-	// initializing the locks.
-	initNSLock(false)
-
-	// Iterate over the cases and assert the result.
-	for i, testCase := range testCases {
-		param := nsParam{testCase.volume, testCase.path}
-		// status of the lock to be set to "Blocked", before setting Blocked->Running.
-		if testCase.setBlocked {
-			globalNSMutex.lockMapMutex.Lock()
-			err := globalNSMutex.statusNoneToBlocked(param, testCase.lockSource, testCase.opsID, testCase.readLock)
-			if err != nil {
-				t.Fatalf("Test %d: Initializing the initial state to Blocked failed <ERROR> %s", i+1, err)
-			}
-			globalNSMutex.lockMapMutex.Unlock()
-		}
-		// invoking the method under test.
-		actualErr = globalNSMutex.statusBlockedToRunning(param, testCase.lockSource, testCase.opsID, testCase.readLock)
-		if actualErr != testCase.expectedErr {
-			t.Fatalf("Test %d: Errors mismatch: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, actualErr)
-		}
-		// In case of no error proceed with validating the lock state information.
-		if actualErr == nil {
-			// debug entry for given <volume, path> pair should exist.
-			if debugLockMap, ok := globalNSMutex.debugLockMap[param]; ok {
-				if lockInfo, ok := debugLockMap.lockInfo[testCase.opsID]; ok {
-					// Validating the lock type filed in the debug lock information.
-					if testCase.readLock {
-						if lockInfo.lType != debugRLockStr {
-							t.Errorf("Test case %d: Expected the lock type in the lock debug info to be \"%s\"", i+1, debugRLockStr)
-						}
-					} else {
-						if lockInfo.lType != debugWLockStr {
-							t.Errorf("Test case %d: Expected the lock type in the lock debug info to be \"%s\"", i+1, debugWLockStr)
-						}
-					}
-
-					// validating the lock origin.
-					if testCase.lockSource != lockInfo.lockSource {
-						t.Errorf("Test %d: Expected the lock origin info to be \"%s\", but got \"%s\"", i+1, testCase.lockSource, lockInfo.lockSource)
-					}
-					// validating the status of the lock.
-					if lockInfo.status != runningStatus {
-						t.Errorf("Test %d: Expected the status of the lock to be \"%s\", but got \"%s\"", i+1, "Running", lockInfo.status)
-					}
-				} else {
-					// Stop the tests if lock debug entry for given <volume, path> pair is not found.
-					t.Fatalf("Test case %d: Expected an debug lock entry for opsID \"%s\"", i+1, testCase.opsID)
-				}
-			} else {
-				// To change the status the entry for given <volume, path> should exist in the lock info struct.
-				t.Fatalf("Test case %d: Debug lock entry for  volume: %s, path: %s doesn't exist", i+1, param.volume, param.path)
-			}
-		}
-	}
-
-}
-
-// TestNsLockMapStatusNoneToBlocked - Validates the function for changing the lock state to blocked
-func TestNsLockMapStatusNoneToBlocked(t *testing.T) {
-
-	testCases := []lockStateCase{
-		// Test case - 1.
-		{
-
-			volume:     "my-bucket",
-			path:       "my-object",
-			lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-			opsID:      "abcd1234",
-			readLock:   true,
-			// expected metrics.
-			expectedErr:        nil,
-			expectedLockStatus: blockedStatus,
-
-			expectedGlobalLockCount:  1,
-			expectedRunningLockCount: 0,
-			expectedBlockedLockCount: 1,
-
-			expectedVolPathLockCount:    1,
-			expectedVolPathRunningCount: 0,
-			expectedVolPathBlockCount:   1,
-		},
-		// Test case - 2.
-		// No entry for <volume, path> pair.
-		// So an attempt to change the state of the lock from `Blocked`->`Running` should fail.
-		{
-
-			volume:     "my-bucket",
-			path:       "my-object-2",
-			lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-			opsID:      "abcd1234",
-			readLock:   false,
-			// expected metrics.
-			expectedErr:        nil,
-			expectedLockStatus: blockedStatus,
-
-			expectedGlobalLockCount:  2,
-			expectedRunningLockCount: 0,
-			expectedBlockedLockCount: 2,
-
-			expectedVolPathLockCount:    1,
-			expectedVolPathRunningCount: 0,
-			expectedVolPathBlockCount:   1,
-		},
-		// Test case - 3.
-		// Entry for the given operationID doesn't exist in the lock state info.
-		// The entry should be created and relevant counters should be set.
-		{
-			volume:     "my-bucket",
-			path:       "my-object",
-			lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-			opsID:      "ops-Id-not-registered",
-			readLock:   true,
-			// expected metrics.
-			expectedErr:        nil,
-			expectedLockStatus: "Blocked",
-
-			expectedGlobalLockCount:  3,
-			expectedRunningLockCount: 0,
-			expectedBlockedLockCount: 3,
-
-			expectedVolPathLockCount:    2,
-			expectedVolPathRunningCount: 0,
-			expectedVolPathBlockCount:   2,
-		},
-	}
-
-	// initializing the locks.
-	initNSLock(false)
-
-	param := nsParam{testCases[0].volume, testCases[0].path}
-	// Testing before the initialization done.
-	// Since the data structures for
-	actualErr := globalNSMutex.statusBlockedToRunning(param, testCases[0].lockSource,
-		testCases[0].opsID, testCases[0].readLock)
-
-	expectedErr := LockInfoVolPathMissing{testCases[0].volume, testCases[0].path}
-	if actualErr != expectedErr {
-		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedErr, actualErr)
-	}
-
-	// Iterate over the cases and assert the result.
-	for i, testCase := range testCases {
-		globalNSMutex.lockMapMutex.Lock()
-		param := nsParam{testCase.volume, testCase.path}
-		actualErr := globalNSMutex.statusNoneToBlocked(param, testCase.lockSource, testCase.opsID, testCase.readLock)
-		if actualErr != testCase.expectedErr {
-			t.Fatalf("Test %d: Errors mismatch: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, actualErr)
-		}
-		globalNSMutex.lockMapMutex.Unlock()
-		if actualErr == nil {
-			verifyLockState(testCase, t, i+1)
-		}
-	}
-}
-
-// TestNsLockMapDeleteLockInfoEntryForOps - Validates the removal of entry for given Operational ID from the lock info.
-func TestNsLockMapDeleteLockInfoEntryForOps(t *testing.T) {
-	testCases := []lockStateCase{
-		// Test case - 1.
-		{
-			volume:     "my-bucket",
-			path:       "my-object",
-			lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-			opsID:      "abcd1234",
-			readLock:   true,
-			// expected metrics.
-		},
-	}
-
-	// initializing the locks.
-	initNSLock(false)
-
-	// case  - 1.
-	// Testing the case where delete lock info is attempted even before the lock is initialized.
-	param := nsParam{testCases[0].volume, testCases[0].path}
-	// Testing before the initialization done.
-
-	actualErr := globalNSMutex.deleteLockInfoEntryForOps(param, testCases[0].opsID)
-
-	expectedErr := LockInfoVolPathMissing{testCases[0].volume, testCases[0].path}
-	if actualErr != expectedErr {
-		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedErr, actualErr)
-	}
-
-	// Case - 2.
-	// Lock state is set to Running and then an attempt to delete the info for non-existent opsID done.
-	globalNSMutex.lockMapMutex.Lock()
-	err := globalNSMutex.statusNoneToBlocked(param, testCases[0].lockSource, testCases[0].opsID, testCases[0].readLock)
-	if err != nil {
-		t.Fatalf("Setting lock status to Blocked failed: <ERROR> %s", err)
-	}
-	globalNSMutex.lockMapMutex.Unlock()
-	err = globalNSMutex.statusBlockedToRunning(param, testCases[0].lockSource, testCases[0].opsID, testCases[0].readLock)
-	if err != nil {
-		t.Fatalf("Setting lock status to Running failed: <ERROR> %s", err)
-	}
-	actualErr = globalNSMutex.deleteLockInfoEntryForOps(param, "non-existent-OpsID")
-
-	expectedOpsIDErr := LockInfoOpsIDNotFound{param.volume, param.path, "non-existent-OpsID"}
-	if actualErr != expectedOpsIDErr {
-		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedOpsIDErr, actualErr)
-	}
-	// case - 4.
-	// Attempt to delete an registered entry is done.
-	// All metrics should be 0 after deleting the entry.
-
-	// Verify that the entry the opsID exists.
-	if debugLockMap, ok := globalNSMutex.debugLockMap[param]; ok {
-		if _, ok := debugLockMap.lockInfo[testCases[0].opsID]; !ok {
-			t.Fatalf("Entry for OpsID \"%s\" in <volume> %s, <path> %s should have existed. ", testCases[0].opsID, param.volume, param.path)
-		}
-	} else {
-		t.Fatalf("Entry for <volume> %s, <path> %s should have existed. ", param.volume, param.path)
-	}
-
-	actualErr = globalNSMutex.deleteLockInfoEntryForOps(param, testCases[0].opsID)
-	if actualErr != nil {
-		t.Fatalf("Expected the error to be <nil>, but got <ERROR> %s", actualErr)
-	}
-
-	// Verify that the entry for the opsId doesn't exists.
-	if debugLockMap, ok := globalNSMutex.debugLockMap[param]; ok {
-		if _, ok := debugLockMap.lockInfo[testCases[0].opsID]; ok {
-			t.Fatalf("The entry for opsID \"%s\" should have been deleted", testCases[0].opsID)
-		}
-	} else {
-		t.Fatalf("Entry for <volume> %s, <path> %s should have existed. ", param.volume, param.path)
-	}
-	if globalNSMutex.counters.granted != 0 {
-		t.Errorf("Expected the count of total running locks to be %v, but got %v", 0, globalNSMutex.counters.granted)
-	}
-	if globalNSMutex.counters.blocked != 0 {
-		t.Errorf("Expected the count of total blocked locks to be %v, but got %v", 0, globalNSMutex.counters.blocked)
-	}
-	if globalNSMutex.counters.total != 0 {
-		t.Errorf("Expected the count of all locks to be %v, but got %v", 0, globalNSMutex.counters.total)
-	}
-}
-
-// TestNsLockMapDeleteLockInfoEntryForVolumePath - Tests validate the logic for removal
-// of entry for given <volume, path> pair from lock info.
-func TestNsLockMapDeleteLockInfoEntryForVolumePath(t *testing.T) {
-	testCases := []lockStateCase{
-		// Test case - 1.
-		{
-			volume:     "my-bucket",
-			path:       "my-object",
-			lockSource: "/home/vadmeste/work/go/src/github.com/minio/minio/xl-v1-object.go:683 +0x2a",
-			opsID:      "abcd1234",
-			readLock:   true,
-			// expected metrics.
-		},
-	}
-
-	// initializing the locks.
-	initNSLock(false)
-
-	// case  - 1.
-	// Case where an attempt to delete the entry for non-existent <volume, path> pair is done.
-	// Set the status of the lock to blocked and then to running.
-	param := nsParam{testCases[0].volume, testCases[0].path}
-	actualErr := globalNSMutex.deleteLockInfoEntryForVolumePath(param)
-	expectedNilErr := LockInfoVolPathMissing{param.volume, param.path}
-	if actualErr != expectedNilErr {
-		t.Fatalf("Errors mismatch: Expected \"%s\", got \"%s\"", expectedNilErr, actualErr)
-	}
-
-	// case - 2.
-	// Attempt to delete an registered entry is done.
-	// All metrics should be 0 after deleting the entry.
-
-	// Registering the entry first.
-	globalNSMutex.lockMapMutex.Lock()
-	err := globalNSMutex.statusNoneToBlocked(param, testCases[0].lockSource, testCases[0].opsID, testCases[0].readLock)
-	if err != nil {
-		t.Fatalf("Setting lock status to Blocked failed: <ERROR> %s", err)
-	}
-	globalNSMutex.lockMapMutex.Unlock()
-	err = globalNSMutex.statusBlockedToRunning(param, testCases[0].lockSource, testCases[0].opsID, testCases[0].readLock)
-	if err != nil {
-		t.Fatalf("Setting lock status to Running failed: <ERROR> %s", err)
-	}
-	// Verify that the entry the for given <volume, path> exists.
-	if _, ok := globalNSMutex.debugLockMap[param]; !ok {
-		t.Fatalf("Entry for <volume> %s, <path> %s should have existed.", param.volume, param.path)
-	}
-	// first delete the entry for the operation ID.
-	_ = globalNSMutex.deleteLockInfoEntryForOps(param, testCases[0].opsID)
-	actualErr = globalNSMutex.deleteLockInfoEntryForVolumePath(param)
-	if actualErr != nil {
-		t.Fatalf("Expected the error to be <nil>, but got <ERROR> %s", actualErr)
-	}
-
-	// Verify that the entry for the opsId doesn't exists.
-	if _, ok := globalNSMutex.debugLockMap[param]; ok {
-		t.Fatalf("Entry for <volume> %s, <path> %s should have been deleted. ", param.volume, param.path)
-	}
-	// The lock count values should be 0.
-	if globalNSMutex.counters.granted != 0 {
-		t.Errorf("Expected the count of total running locks to be %v, but got %v", 0, globalNSMutex.counters.granted)
-	}
-	if globalNSMutex.counters.blocked != 0 {
-		t.Errorf("Expected the count of total blocked locks to be %v, but got %v", 0, globalNSMutex.counters.blocked)
-	}
-	if globalNSMutex.counters.total != 0 {
-		t.Errorf("Expected the count of all locks to be %v, but got %v", 0, globalNSMutex.counters.total)
-	}
-}
-
-// Test to assert that status change from blocked to none shouldn't remove lock info entry for ops
-// Ref: Logs from https://github.com/minio/minio/issues/5311
-func TestStatusBlockedToNone(t *testing.T) {
-	// Initialize namespace lock subsystem
-	initNSLock(false)
-
-	ns := globalNSMutex
-
-	volume, path := "bucket", "object"
-	param := nsParam{volume: volume, path: path}
-	lockSrc := "main.go:1"
-	opsID := "1"
-
-	err := ns.statusNoneToBlocked(param, lockSrc, opsID, false)
-	if err != nil {
-		t.Fatal("Failed to mark lock state to blocked")
-	}
-
-	err = ns.statusBlockedToNone(param, lockSrc, opsID, false)
-	if err != nil {
-		t.Fatal("Failed to mark lock state to none")
-	}
-
-	err = ns.deleteLockInfoEntryForOps(param, opsID)
-	if err != nil {
-		t.Fatalf("Expected deleting of lock entry for %s to pass but got %v", opsID, err)
-	}
-
-}

cmd/lockinfo-handlers.go

@@ -1,67 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"time"
-)
-
-// SystemLockState - Structure to fill the lock state of entire object storage.
-// That is the total locks held, total calls blocked on locks and state of all the locks for the entire system.
-type SystemLockState struct {
-	TotalLocks int64 `json:"totalLocks"`
-	// Count of operations which are blocked waiting for the lock to
-	// be released.
-	TotalBlockedLocks int64 `json:"totalBlockedLocks"`
-	// Count of operations which has successfully acquired the lock but
-	// hasn't unlocked yet (operation in progress).
-	TotalAcquiredLocks int64            `json:"totalAcquiredLocks"`
-	LocksInfoPerObject []VolumeLockInfo `json:"locksInfoPerObject"`
-}
-
-// VolumeLockInfo - Structure to contain the lock state info for volume, path pair.
-type VolumeLockInfo struct {
-	Bucket string `json:"bucket"`
-	Object string `json:"object"`
-
-	// All locks blocked + running for given <volume,path> pair.
-	LocksOnObject int64 `json:"-"`
-	// Count of operations which has successfully acquired the lock
-	// but hasn't unlocked yet( operation in progress).
-	LocksAcquiredOnObject int64 `json:"-"`
-	// Count of operations which are blocked waiting for the lock
-	// to be released.
-	TotalBlockedLocks int64 `json:"-"`
-
-	// Count of all read locks
-	TotalReadLocks int64 `json:"readLocks"`
-	// Count of all write locks
-	TotalWriteLocks int64 `json:"writeLocks"`
-	// State information containing state of the locks for all operations
-	// on given <volume,path> pair.
-	LockDetailsOnObject []OpsLockState `json:"lockOwners"`
-}
-
-// OpsLockState - structure to fill in state information of the lock.
-// structure to fill in status information for each operation with given operation ID.
-type OpsLockState struct {
-	OperationID string     `json:"id"`     // String containing operation ID.
-	LockSource  string     `json:"source"` // Operation type (GetObject, PutObject...)
-	LockType    lockType   `json:"type"`   // Lock type (RLock, WLock)
-	Status      statusType `json:"status"` // Status can be Running/Ready/Blocked.
-	Since       time.Time  `json:"since"`  // Time when the lock was initially held.
-}

cmd/logger/console.go

@@ -0,0 +1,101 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// ConsoleTarget implements loggerTarget to send log
+// in plain or json format to the standard output.
+type ConsoleTarget struct{}
+
+func (c *ConsoleTarget) send(entry logEntry) error {
+	if jsonFlag {
+		logJSON, err := json.Marshal(&entry)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(logJSON))
+		return nil
+	}
+
+	trace := make([]string, len(entry.Trace.Source))
+
+	// Add a sequence number and formatting for each stack trace
+	// No formatting is required for the first entry
+	for i, element := range entry.Trace.Source {
+		trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
+	}
+
+	tagString := ""
+	for key, value := range entry.Trace.Variables {
+		if value != "" {
+			if tagString != "" {
+				tagString += ", "
+			}
+			tagString += key + "=" + value
+		}
+	}
+
+	apiString := "API: " + entry.API.Name + "("
+	if entry.API.Args != nil && entry.API.Args.Bucket != "" {
+		apiString = apiString + "bucket=" + entry.API.Args.Bucket
+	}
+	if entry.API.Args != nil && entry.API.Args.Object != "" {
+		apiString = apiString + ", object=" + entry.API.Args.Object
+	}
+	apiString += ")"
+	timeString := "Time: " + time.Now().Format(loggerTimeFormat)
+
+	var requestID string
+	if entry.RequestID != "" {
+		requestID = "\nRequestID: " + entry.RequestID
+	}
+
+	var remoteHost string
+	if entry.RemoteHost != "" {
+		remoteHost = "\nRemoteHost: " + entry.RemoteHost
+	}
+
+	var userAgent string
+	if entry.UserAgent != "" {
+		userAgent = "\nUserAgent: " + entry.UserAgent
+	}
+
+	if len(entry.Trace.Variables) > 0 {
+		tagString = "\n       " + tagString
+	}
+
+	var msg = colorFgRed(colorBold(entry.Trace.Message))
+	var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
+		apiString, timeString, requestID, remoteHost, userAgent,
+		msg, tagString, strings.Join(trace, "\n"))
+
+	fmt.Println(output)
+	return nil
+}
+
+// NewConsole initializes a new logger target
+// which prints log directly in the standard
+// output.
+func NewConsole() LoggingTarget {
+	return &ConsoleTarget{}
+}

cmd/logger/http.go

@@ -0,0 +1,88 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"net/http"
+)
+
+// HTTPTarget implements loggerTarget and sends the json
+// format of a log entry to the configured http endpoint.
+// An internal buffer of logs is maintained but when the
+// buffer is full, new logs are just ignored and an error
+// is returned to the caller.
+type HTTPTarget struct {
+	// Channel of log entries
+	logCh chan logEntry
+	// HTTP(s) endpoint
+	endpoint string
+	client   http.Client
+}
+
+func (h *HTTPTarget) startHTTPLogger() {
+	// Create a routine which sends json logs received
+	// from an internal channel.
+	go func() {
+		for entry := range h.logCh {
+			logJSON, err := json.Marshal(&entry)
+			if err != nil {
+				continue
+			}
+
+			req, err := http.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
+			req.Header.Set("Content-Type", "application/json")
+
+			resp, err := h.client.Do(req)
+			if err != nil {
+				continue
+			}
+			if resp.Body != nil {
+				resp.Body.Close()
+			}
+		}
+	}()
+}
+
+// NewHTTP initializes a new logger target which
+// sends log over http to the specified endpoint
+func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
+	h := HTTPTarget{
+		endpoint: endpoint,
+		client: http.Client{
+			Transport: transport,
+		},
+		logCh: make(chan logEntry, 10000),
+	}
+
+	h.startHTTPLogger()
+	return &h
+}
+
+func (h *HTTPTarget) send(entry logEntry) error {
+	select {
+	case h.logCh <- entry:
+	default:
+		// log channel is full, do not wait and return
+		// an error immediately to the caller
+		return errors.New("log buffer full")
+	}
+
+	return nil
+}

cmd/logger/logger.go

@@ -28,7 +28,6 @@ import (
 	"time"
 
 	c "github.com/minio/mc/pkg/console"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 // Disable disables all logging, false by default. (used for "go test")
@@ -122,14 +121,15 @@ type api struct {
 }
 
 type logEntry struct {
-	Level      string      `json:"level"`
-	Time       string      `json:"time"`
-	API        *api        `json:"api,omitempty"`
-	RemoteHost string      `json:"remotehost,omitempty"`
-	RequestID  string      `json:"requestID,omitempty"`
-	UserAgent  string      `json:"userAgent,omitempty"`
-	Message    string      `json:"message,omitempty"`
-	Trace      *traceEntry `json:"error,omitempty"`
+	DeploymentID string      `json:"deploymentid,omitempty"`
+	Level        string      `json:"level"`
+	Time         string      `json:"time"`
+	API          *api        `json:"api,omitempty"`
+	RemoteHost   string      `json:"remotehost,omitempty"`
+	RequestID    string      `json:"requestID,omitempty"`
+	UserAgent    string      `json:"userAgent,omitempty"`
+	Message      string      `json:"message,omitempty"`
+	Trace        *traceEntry `json:"error,omitempty"`
 }
 
 // quiet: Hide startup messages if enabled
@@ -138,8 +138,15 @@ var (
 	quiet, jsonFlag bool
 	// Custom function to format error
 	errorFmtFunc func(string, error, bool) string
+
+	deploymentID string
 )
 
+// SetDeploymentID - Used to set the deployment ID, in XL and FS mode
+func SetDeploymentID(id string) {
+	deploymentID = id
+}
+
 // EnableQuiet - turns quiet option on.
 func EnableQuiet() {
 	quiet = true
@@ -292,74 +299,21 @@ func LogIf(ctx context.Context, err error) {
 	// Get the cause for the Error
 	message := err.Error()
 
-	// Output the formatted log message at console
-	var output string
-	if jsonFlag {
-		logJSON, err := json.Marshal(&logEntry{
-			Level:      ErrorLvl.String(),
-			RemoteHost: req.RemoteHost,
-			RequestID:  req.RequestID,
-			UserAgent:  req.UserAgent,
-			Time:       time.Now().UTC().Format(time.RFC3339Nano),
-			API:        &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
-			Trace:      &traceEntry{Message: message, Source: trace, Variables: tags},
-		})
-		if err != nil {
-			panic(err)
-		}
-		output = string(logJSON)
-	} else {
-		// Add a sequence number and formatting for each stack trace
-		// No formatting is required for the first entry
-		for i, element := range trace {
-			trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
-		}
-
-		tagString := ""
-		for key, value := range tags {
-			if value != "" {
-				if tagString != "" {
-					tagString += ", "
-				}
-				tagString += key + "=" + value
-			}
-		}
-
-		apiString := "API: " + API + "("
-		if req.BucketName != "" {
-			apiString = apiString + "bucket=" + req.BucketName
-		}
-		if req.ObjectName != "" {
-			apiString = apiString + ", object=" + req.ObjectName
-		}
-		apiString += ")"
-		timeString := "Time: " + time.Now().Format(loggerTimeFormat)
-
-		var requestID string
-		if req.RequestID != "" {
-			requestID = "\nRequestID: " + req.RequestID
-		}
-
-		var remoteHost string
-		if req.RemoteHost != "" {
-			remoteHost = "\nRemoteHost: " + req.RemoteHost
-		}
-
-		var userAgent string
-		if req.UserAgent != "" {
-			userAgent = "\nUserAgent: " + req.UserAgent
-		}
-
-		if len(tags) > 0 {
-			tagString = "\n       " + tagString
-		}
-
-		var msg = colorFgRed(colorBold(message))
-		output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
-			apiString, timeString, requestID, remoteHost, userAgent,
-			msg, tagString, strings.Join(trace, "\n"))
+	entry := logEntry{
+		DeploymentID: deploymentID,
+		Level:        ErrorLvl.String(),
+		RemoteHost:   req.RemoteHost,
+		RequestID:    req.RequestID,
+		UserAgent:    req.UserAgent,
+		Time:         time.Now().UTC().Format(time.RFC3339Nano),
+		API:          &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
+		Trace:        &traceEntry{Message: message, Source: trace, Variables: tags},
+	}
+
+	// Iterate over all logger targets to send the log entry
+	for _, t := range Targets {
+		t.send(entry)
 	}
-	fmt.Println(output)
 }
 
 // ErrCritical is the value panic'd whenever CriticalIf is called.
@@ -423,23 +377,15 @@ func (f fatalMsg) quiet(msg string, args ...interface{}) {
 }
 
 var (
-	logTag       = "ERROR"
-	logBanner    = colorBgRed(colorFgWhite(colorBold(logTag))) + " "
-	emptyBanner  = colorBgRed(strings.Repeat(" ", len(logTag))) + " "
-	minimumWidth = 80
-	bannerWidth  = len(logTag) + 1
+	logTag      = "ERROR"
+	logBanner   = colorBgRed(colorFgWhite(colorBold(logTag))) + " "
+	emptyBanner = colorBgRed(strings.Repeat(" ", len(logTag))) + " "
+	bannerWidth = len(logTag) + 1
 )
 
 func (f fatalMsg) pretty(msg string, args ...interface{}) {
 	// Build the passed error message
 	errMsg := fmt.Sprintf(msg, args...)
-	// Check terminal width
-	termWidth, _, err := terminal.GetSize(0)
-	if err != nil || termWidth < minimumWidth {
-		termWidth = minimumWidth
-	}
-	// Calculate available widht without the banner
-	width := termWidth - bannerWidth
 
 	tagPrinted := false
 
@@ -470,13 +416,8 @@ func (f fatalMsg) pretty(msg string, args ...interface{}) {
 			ansiRestoreAttributes()
 			ansiMoveRight(bannerWidth)
 			// Continue  error message printing
-			if len(line) > width {
-				fmt.Println(line[:width])
-				line = line[width:]
-			} else {
-				fmt.Println(line)
-				break
-			}
+			fmt.Println(line)
+			break
 		}
 	}
 

cmd/logger/targets.go

@@ -0,0 +1,33 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+// LoggingTarget is the entity that we will receive
+// a single log entry and send it to the log target
+//   e.g. send the log to a http server
+type LoggingTarget interface {
+	send(entry logEntry) error
+}
+
+// Targets is the set of enabled loggers
+var Targets = []LoggingTarget{}
+
+// AddTarget adds a new logger target to the
+// list of enabled loggers
+func AddTarget(t LoggingTarget) {
+	Targets = append(Targets, t)
+}

cmd/namespace-lock.go

@@ -104,10 +104,6 @@ func newNSLock(isDistXL bool) *nsLockMap {
 		lockMap:  make(map[nsParam]*nsLock),
 		counters: &lockStat{},
 	}
-
-	// Initialize nsLockMap with entry for instrumentation information.
-	// Entries of <volume,path> -> stateInfo of locks
-	nsMutex.debugLockMap = make(map[nsParam]*debugLockInfoPerVolumePath)
 	return &nsMutex
 }
 
@@ -132,8 +128,7 @@ type nsLock struct {
 // Unlock, RLock and RUnlock.
 type nsLockMap struct {
 	// Lock counter used for lock debugging.
-	counters     *lockStat
-	debugLockMap map[nsParam]*debugLockInfoPerVolumePath // Info for instrumentation on locks.
+	counters *lockStat
 
 	// Indicates if namespace is part of a distributed setup.
 	isDistXL     bool
@@ -162,12 +157,6 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock
 	}
 	nsLk.ref++ // Update ref count here to avoid multiple races.
 
-	// Change the state of the lock to be blocked for the given
-	// pair of <volume, path> and <OperationID> till the lock
-	// unblocks. The lock for accessing `globalNSMutex` is held inside
-	// the function itself.
-	n.statusNoneToBlocked(param, lockSource, opsID, readLock)
-
 	// Unlock map before Locking NS which might block.
 	n.lockMapMutex.Unlock()
 
@@ -181,28 +170,14 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock
 	if !locked { // We failed to get the lock
 		n.lockMapMutex.Lock()
 		defer n.lockMapMutex.Unlock()
-		// Changing the status of the operation from blocked to none
-		n.statusBlockedToNone(param, lockSource, opsID, readLock)
 
 		nsLk.ref-- // Decrement ref count since we failed to get the lock
-		// delete the lock state entry for given operation ID.
-		n.deleteLockInfoEntryForOps(param, opsID)
 
 		if nsLk.ref == 0 {
 			// Remove from the map if there are no more references.
 			delete(n.lockMap, param)
-
-			// delete the lock state entry for given
-			// <volume, path> pair.
-			n.deleteLockInfoEntryForVolumePath(param)
 		}
-		return
 	}
-
-	// Changing the status of the operation from blocked to
-	// running.  change the state of the lock to be running (from
-	// blocked) for the given pair of <volume, path> and <OperationID>.
-	n.statusBlockedToRunning(param, lockSource, opsID, readLock)
 	return
 }
 
@@ -225,17 +200,10 @@ func (n *nsLockMap) unlock(volume, path, opsID string, readLock bool) {
 		}
 		if nsLk.ref != 0 {
 			nsLk.ref--
-
-			// delete the lock state entry for given operation ID.
-			n.deleteLockInfoEntryForOps(param, opsID)
 		}
 		if nsLk.ref == 0 {
 			// Remove from the map if there are no more references.
 			delete(n.lockMap, param)
-
-			// delete the lock state entry for given
-			// <volume, path> pair.
-			n.deleteLockInfoEntryForVolumePath(param)
 		}
 	}
 }
@@ -296,11 +264,6 @@ func (n *nsLockMap) ForceUnlock(volume, path string) {
 		// Remove lock from the map.
 		delete(n.lockMap, param)
 	}
-
-	// delete the lock state entry for given
-	// <volume, path> pair. Ignore error as there
-	// is no way to report it back
-	n.deleteLockInfoEntryForVolumePath(param)
 }
 
 // lockInstance - frontend/top-level interface for namespace locks.
@@ -313,7 +276,8 @@ type lockInstance struct {
 // path. The returned lockInstance object encapsulates the nsLockMap,
 // volume, path and operation ID.
 func (n *nsLockMap) NewNSLock(volume, path string) RWLocker {
-	return &lockInstance{n, volume, path, getOpsID()}
+	opsID := mustGetUUID()
+	return &lockInstance{n, volume, path, opsID}
 }
 
 // Lock - block until write lock is taken or timeout has occurred.

cmd/notification.go

@@ -429,12 +429,8 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
 }
 
 // NewNotificationSys - creates new notification system object.
-func NewNotificationSys(config *serverConfig, endpoints EndpointList) (*NotificationSys, error) {
-	targetList, err := getNotificationTargets(config)
-	if err != nil {
-		return nil, err
-	}
-
+func NewNotificationSys(config *serverConfig, endpoints EndpointList) *NotificationSys {
+	targetList := getNotificationTargets(config)
 	peerRPCClientMap := makeRemoteRPCClients(endpoints)
 
 	// bucketRulesMap/bucketRemoteTargetRulesMap are initialized by NotificationSys.Init()
@@ -443,7 +439,7 @@ func NewNotificationSys(config *serverConfig, endpoints EndpointList) (*Notifica
 		bucketRulesMap:             make(map[string]event.RulesMap),
 		bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
 		peerRPCClientMap:           peerRPCClientMap,
-	}, nil
+	}
 }
 
 type eventArgs struct {
@@ -551,7 +547,7 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*by
 	err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "")
 	if err != nil {
 		// Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
+		if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
 			return nil, errConfigNotFound
 		}
 

cmd/object-api-errors.go

@@ -320,13 +320,6 @@ func (e InvalidPart) Error() string {
 	return "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag."
 }
 
-// PartsSizeUnequal - All parts except the last part should be of the same size
-type PartsSizeUnequal struct{}
-
-func (e PartsSizeUnequal) Error() string {
-	return "All parts except the last part should be of the same size"
-}
-
 // PartTooSmall - error if part size is less than 5MB.
 type PartTooSmall struct {
 	PartSize   int64
@@ -397,3 +390,9 @@ func isErrObjectNotFound(err error) bool {
 	}
 	return false
 }
+
+// isInsufficientReadQuorum - Check if error type is InsufficientReadQuorum.
+func isInsufficientReadQuorum(err error) bool {
+	_, ok := err.(InsufficientReadQuorum)
+	return ok
+}

cmd/object-api-interface.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"context"
 	"io"
-	"time"
 
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
@@ -65,10 +64,6 @@ type ObjectLayer interface {
 	ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error)
 	ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error)
 
-	// Locking operations
-	ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
-	ClearLocks(context.Context, []VolumeLockInfo) error
-
 	// Policy operations
 	SetBucketPolicy(context.Context, string, *policy.Policy) error
 	GetBucketPolicy(context.Context, string) (*policy.Policy, error)

cmd/object-handlers.go

@@ -31,7 +31,6 @@ import (
 	"sort"
 	"strconv"
 
-	etcd "github.com/coreos/etcd/client"
 	"github.com/gorilla/mux"
 	miniogo "github.com/minio/minio-go"
 	"github.com/minio/minio/cmd/logger"
@@ -69,7 +68,7 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
 // This implementation of the GET operation retrieves object. To use GET,
 // you must have READ access to the object.
 func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "GetObject")
+	ctx := newContext(r, w, "GetObject")
 
 	var object, bucket string
 	vars := mux.Vars(r)
@@ -219,7 +218,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 // -----------
 // The HEAD operation retrieves metadata from an object without returning the object itself.
 func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "HeadObject")
+	ctx := newContext(r, w, "HeadObject")
 
 	var object, bucket string
 	vars := mux.Vars(r)
@@ -313,7 +312,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 
 // Extract metadata relevant for an CopyObject operation based on conditional
 // header values specified in X-Amz-Metadata-Directive.
-func getCpObjMetadataFromHeader(ctx context.Context, header http.Header, userMeta map[string]string) (map[string]string, error) {
+func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta map[string]string) (map[string]string, error) {
 	// Make a copy of the supplied metadata to avoid
 	// to change the original one.
 	defaultMeta := make(map[string]string, len(userMeta))
@@ -323,13 +322,13 @@ func getCpObjMetadataFromHeader(ctx context.Context, header http.Header, userMet
 
 	// if x-amz-metadata-directive says REPLACE then
 	// we extract metadata from the input headers.
-	if isMetadataReplace(header) {
-		return extractMetadataFromHeader(ctx, header)
+	if isMetadataReplace(r.Header) {
+		return extractMetadata(ctx, r)
 	}
 
 	// if x-amz-metadata-directive says COPY then we
 	// return the default metadata.
-	if isMetadataCopy(header) {
+	if isMetadataCopy(r.Header) {
 		return defaultMeta, nil
 	}
 
@@ -342,7 +341,7 @@ func getCpObjMetadataFromHeader(ctx context.Context, header http.Header, userMet
 // This implementation of the PUT operation adds an object to a bucket
 // while reading the object from another source.
 func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "CopyObject")
+	ctx := newContext(r, w, "CopyObject")
 
 	vars := mux.Vars(r)
 	dstBucket := vars["bucket"]
@@ -514,7 +513,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 	srcInfo.Writer = writer
 
-	srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r.Header, srcInfo.UserDefined)
+	srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined)
 	if err != nil {
 		pipeWriter.CloseWithError(err)
 		writeErrorResponse(w, ErrInternalError, r.URL)
@@ -639,7 +638,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 // ----------
 // This implementation of the PUT operation adds an object to a bucket.
 func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutObject")
+	ctx := newContext(r, w, "PutObject")
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -699,12 +698,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// Extract metadata to be saved from incoming HTTP header.
-	metadata, err := extractMetadataFromHeader(ctx, r.Header)
+	metadata, err := extractMetadata(ctx, r)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
+
 	if rAuthType == authTypeStreamingSigned {
 		if contentEncoding, ok := metadata["content-encoding"]; ok {
 			contentEncoding = trimAwsChunkedContentEncoding(contentEncoding)
@@ -806,6 +805,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	if api.CacheAPI() != nil && !hasSSECustomerHeader(r.Header) {
 		putObject = api.CacheAPI().PutObject
 	}
+
 	// Create the object..
 	objInfo, err := putObject(ctx, bucket, object, hashReader, metadata)
 	if err != nil {
@@ -845,7 +845,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 // NewMultipartUploadHandler - New multipart upload.
 func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "NewMultipartUpload")
+	ctx := newContext(r, w, "NewMultipartUpload")
 
 	var object, bucket string
 	vars := mux.Vars(r)
@@ -901,7 +901,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	}
 
 	// Extract metadata that needs to be saved.
-	metadata, err := extractMetadataFromHeader(ctx, r.Header)
+	metadata, err := extractMetadata(ctx, r)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
@@ -932,7 +932,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 
 // CopyObjectPartHandler - uploads a part by copying data from an existing object as data source.
 func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "CopyObjectPart")
+	ctx := newContext(r, w, "CopyObjectPart")
 
 	vars := mux.Vars(r)
 	dstBucket := vars["bucket"]
@@ -1037,6 +1037,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 	var writer io.WriteCloser = pipeWriter
 	var reader io.Reader = pipeReader
+	var getLength = length
 	srcInfo.Reader, err = hash.NewReader(reader, length, "", "")
 	if err != nil {
 		pipeWriter.CloseWithError(err)
@@ -1056,7 +1057,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			// Response writer should be limited early on for decryption upto required length,
 			// additionally also skipping mod(offset)64KiB boundaries.
 			writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)
-			writer, startOffset, length, err = DecryptBlocksRequest(writer, r, srcBucket, srcObject, startOffset, length, srcInfo, true)
+			writer, startOffset, getLength, err = DecryptBlocksRequest(writer, r, srcBucket, srcObject, startOffset, length, srcInfo, true)
 			if err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -1076,7 +1077,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 				return
 			}
 
-			// Calculating object encryption key
 			var objectEncryptionKey []byte
 			objectEncryptionKey, err = decryptObjectInfo(key, dstBucket, dstObject, li.UserDefined)
 			if err != nil {
@@ -1085,19 +1085,21 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 				return
 			}
 
-			reader, err = sio.EncryptReader(reader, sio.Config{Key: objectEncryptionKey})
+			var partIDbin [4]byte
+			binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
+
+			mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
+			mac.Write(partIDbin[:])
+			partEncryptionKey := mac.Sum(nil)
+			reader, err = sio.EncryptReader(reader, sio.Config{Key: partEncryptionKey})
 			if err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
 
-			size := length
-			if !sseCopyC {
-				info := ObjectInfo{Size: length}
-				size = info.EncryptedSize()
-			}
-
+			info := ObjectInfo{Size: length}
+			size := info.EncryptedSize()
 			srcInfo.Reader, err = hash.NewReader(reader, size, "", "")
 			if err != nil {
 				pipeWriter.CloseWithError(err)
@@ -1111,7 +1113,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	// Copy source object to destination, if source and destination
 	// object is same then only metadata is updated.
 	partInfo, err := objectAPI.CopyObjectPart(ctx, srcBucket, srcObject, dstBucket,
-		dstObject, uploadID, partID, startOffset, length, srcInfo)
+		dstObject, uploadID, partID, startOffset, getLength, srcInfo)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -1129,7 +1131,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 // PutObjectPartHandler - uploads an incoming part for an ongoing multipart operation.
 func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "PutObjectPart")
+	ctx := newContext(r, w, "PutObjectPart")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1328,7 +1330,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 // AbortMultipartUploadHandler - Abort multipart upload
 func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "AbortMultipartUpload")
+	ctx := newContext(r, w, "AbortMultipartUpload")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1367,7 +1369,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 
 // ListObjectPartsHandler - List object parts
 func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "ListObjectParts")
+	ctx := newContext(r, w, "ListObjectParts")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1407,7 +1409,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 
 // CompleteMultipartUploadHandler - Complete multipart upload.
 func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "CompleteMultipartUpload")
+	ctx := newContext(r, w, "CompleteMultipartUpload")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1516,7 +1518,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 // DeleteObjectHandler - delete an object
 func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, "DeleteObject")
+	ctx := newContext(r, w, "DeleteObject")
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1544,7 +1546,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 	if globalDNSConfig != nil {
 		_, err := globalDNSConfig.Get(bucket)
 		if err != nil {
-			if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
+			if err == dns.ErrNoEntriesFound {
 				writeErrorResponse(w, ErrNoSuchBucket, r.URL)
 			} else {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/object-handlers_test.go

@@ -263,7 +263,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey), getGetObjectURL("", bucketName, "abcd"))),
+			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey), getGetObjectURL("", bucketName, "abcd"), "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 3.
@@ -287,7 +287,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidRange), getGetObjectURL("", bucketName, objectName))),
+			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidRange), getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusRequestedRangeNotSatisfiable,
 		},
 		// Test case - 5.
@@ -313,7 +313,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  "Invalid-AccessID",
 			secretKey:  credentials.SecretKey,
 
-			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID), getGetObjectURL("", bucketName, objectName))),
+			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID), getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusForbidden,
 		},
 		// Test case - 7.
@@ -326,7 +326,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			secretKey:  credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidObjectName),
-				getGetObjectURL("", bucketName, "../../etc"))),
+				getGetObjectURL("", bucketName, "../../etc"), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 8.
@@ -339,7 +339,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			secretKey:  credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey),
-				"/"+bucketName+"/"+". ./. ./etc")),
+				"/"+bucketName+"/"+". ./. ./etc", "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 9.
@@ -352,7 +352,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			secretKey:  credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidObjectName),
-				"/"+bucketName+"/"+". ./../etc")),
+				"/"+bucketName+"/"+". ./../etc", "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 10.
@@ -365,7 +365,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			secretKey:  credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey),
-				getGetObjectURL("", bucketName, "etc/path/proper/.../etc"))),
+				getGetObjectURL("", bucketName, "etc/path/proper/.../etc"), "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 	}
@@ -2238,7 +2238,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidPart{})),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 2.
@@ -2253,7 +2253,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrMalformedXML),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 3.
@@ -2268,7 +2268,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidUploadID{UploadID: "abc"})),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 4.
@@ -2283,7 +2283,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 
 			expectedContent: encodeResponse(completeMultipartAPIError{int64(4), int64(5242880), 1, "e2fc714c4727ee9395f324cd2e7f331f",
 				getAPIErrorResponse(getAPIError(toAPIErrorCode(PartTooSmall{PartNumber: 1})),
-					getGetObjectURL("", bucketName, objectName))}),
+					getGetObjectURL("", bucketName, objectName), "")}),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 5.
@@ -2297,7 +2297,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidPart{})),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 6.
@@ -2312,7 +2312,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidPartOrder),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 7.
@@ -2327,7 +2327,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID),
-				getGetObjectURL("", bucketName, objectName))),
+				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusForbidden,
 		},
 		// Test case - 8.

cmd/posix-errors.go

@@ -24,26 +24,65 @@ import (
 
 // Function not implemented error
 func isSysErrNoSys(err error) bool {
-	return err == syscall.ENOSYS
+	if err == syscall.ENOSYS {
+		return true
+	}
+	if pathErr, ok := err.(*os.PathError); ok {
+		switch pathErr.Err {
+		case syscall.ENOSYS:
+			return true
+		}
+	}
+	return false
 }
 
 // Not supported error
 func isSysErrOpNotSupported(err error) bool {
-	return err == syscall.EOPNOTSUPP
+	if err == syscall.EOPNOTSUPP {
+		return true
+	}
+	if pathErr, ok := err.(*os.PathError); ok {
+		switch pathErr.Err {
+		case syscall.EOPNOTSUPP:
+			return true
+		}
+	}
+	return false
 }
 
 // No space left on device error
 func isSysErrNoSpace(err error) bool {
-	return err == syscall.ENOSPC
+	if err == syscall.ENOSPC {
+		return true
+	}
+	if pathErr, ok := err.(*os.PathError); ok {
+		switch pathErr.Err {
+		case syscall.ENOSPC:
+			return true
+		}
+	}
+	return false
 }
 
 // Input/output error
 func isSysErrIO(err error) bool {
-	return err == syscall.EIO
+	if err == syscall.EIO {
+		return true
+	}
+	if pathErr, ok := err.(*os.PathError); ok {
+		switch pathErr.Err {
+		case syscall.EIO:
+			return true
+		}
+	}
+	return false
 }
 
 // Check if the given error corresponds to EISDIR (is a directory).
 func isSysErrIsDir(err error) bool {
+	if err == syscall.EISDIR {
+		return true
+	}
 	if pathErr, ok := err.(*os.PathError); ok {
 		switch pathErr.Err {
 		case syscall.EISDIR:
@@ -55,6 +94,9 @@ func isSysErrIsDir(err error) bool {
 
 // Check if the given error corresponds to ENOTDIR (is not a directory).
 func isSysErrNotDir(err error) bool {
+	if err == syscall.ENOTDIR {
+		return true
+	}
 	if pathErr, ok := err.(*os.PathError); ok {
 		switch pathErr.Err {
 		case syscall.ENOTDIR:
@@ -66,6 +108,9 @@ func isSysErrNotDir(err error) bool {
 
 // Check if the given error corresponds to the ENAMETOOLONG (name too long).
 func isSysErrTooLong(err error) bool {
+	if err == syscall.ENAMETOOLONG {
+		return true
+	}
 	if pathErr, ok := err.(*os.PathError); ok {
 		switch pathErr.Err {
 		case syscall.ENAMETOOLONG:
@@ -78,6 +123,9 @@ func isSysErrTooLong(err error) bool {
 // Check if the given error corresponds to ENOTEMPTY for unix
 // and ERROR_DIR_NOT_EMPTY for windows (directory not empty).
 func isSysErrNotEmpty(err error) bool {
+	if err == syscall.ENOTEMPTY {
+		return true
+	}
 	if pathErr, ok := err.(*os.PathError); ok {
 		if runtime.GOOS == globalWindowsOSName {
 			if errno, _ok := pathErr.Err.(syscall.Errno); _ok && errno == 0x91 {

cmd/posix-list-dir-nix.go

@@ -22,7 +22,6 @@ import (
 	"os"
 	"path"
 	"runtime"
-	"strings"
 	"sync"
 	"syscall"
 	"unsafe"
@@ -124,18 +123,12 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 
 	d, err := os.Open(dirPath)
 	if err != nil {
-		// File is really not found.
-		if os.IsNotExist(err) {
+		if os.IsNotExist(err) || isSysErrNotDir(err) {
 			return nil, errFileNotFound
 		}
 		if os.IsPermission(err) {
 			return nil, errFileAccessDenied
 		}
-
-		// File path cannot be verified since one of the parents is a file.
-		if strings.Contains(err.Error(), "not a directory") {
-			return nil, errFileNotFound
-		}
 		return nil, err
 	}
 	defer d.Close()
@@ -148,6 +141,9 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 	for !done {
 		nbuf, err := syscall.ReadDirent(fd, buf)
 		if err != nil {
+			if isSysErrNotDir(err) {
+				return nil, errFileNotFound
+			}
 			return nil, err
 		}
 		if nbuf <= 0 {

cmd/posix.go

@@ -327,15 +327,18 @@ func (s *posix) checkDiskFound() (err error) {
 	if !s.IsOnline() {
 		return errDiskNotFound
 	}
-	_, err = os.Stat((s.diskPath))
+	_, err = os.Stat(s.diskPath)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errDiskNotFound
 		} else if isSysErrTooLong(err) {
 			return errFileNameTooLong
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
+		return err
 	}
-	return err
+	return nil
 }
 
 // diskUsage returns du information for the posix path, in a continuous routine.
@@ -421,7 +424,7 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 // Make a volume entry.
 func (s *posix) MakeVol(volume string) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -451,6 +454,8 @@ func (s *posix) MakeVol(volume string) (err error) {
 		}
 		if os.IsPermission(err) {
 			return errDiskAccessDenied
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 		return err
 	}
@@ -462,7 +467,7 @@ func (s *posix) MakeVol(volume string) (err error) {
 // ListVols - list volumes.
 func (s *posix) ListVols() (volsInfo []VolInfo, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -475,8 +480,11 @@ func (s *posix) ListVols() (volsInfo []VolInfo, err error) {
 		return nil, err
 	}
 
-	volsInfo, err = listVols((s.diskPath))
+	volsInfo, err = listVols(s.diskPath)
 	if err != nil {
+		if isSysErrIO(err) {
+			return nil, errFaultyDisk
+		}
 		return nil, err
 	}
 	for i, vol := range volsInfo {
@@ -505,11 +513,13 @@ func listVols(dirPath string) ([]VolInfo, error) {
 			continue
 		}
 		var fi os.FileInfo
-		fi, err = os.Stat((pathJoin(dirPath, entry)))
+		fi, err = os.Stat(pathJoin(dirPath, entry))
 		if err != nil {
 			// If the file does not exist, skip the entry.
 			if os.IsNotExist(err) {
 				continue
+			} else if isSysErrIO(err) {
+				return nil, errFaultyDisk
 			}
 			return nil, err
 		}
@@ -526,7 +536,7 @@ func listVols(dirPath string) ([]VolInfo, error) {
 // StatVol - get volume info.
 func (s *posix) StatVol(volume string) (volInfo VolInfo, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -550,6 +560,8 @@ func (s *posix) StatVol(volume string) (volInfo VolInfo, err error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			return VolInfo{}, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return VolInfo{}, errFaultyDisk
 		}
 		return VolInfo{}, err
 	}
@@ -565,7 +577,7 @@ func (s *posix) StatVol(volume string) (volInfo VolInfo, err error) {
 // DeleteVol - delete a volume.
 func (s *posix) DeleteVol(volume string) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -591,8 +603,9 @@ func (s *posix) DeleteVol(volume string) (err error) {
 			return errVolumeNotEmpty
 		} else if os.IsPermission(err) {
 			return errDiskAccessDenied
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
-
 		return err
 	}
 	return nil
@@ -602,7 +615,7 @@ func (s *posix) DeleteVol(volume string) (err error) {
 // If an entry is a directory it will be returned with a trailing "/".
 func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -625,6 +638,8 @@ func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, er
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -644,7 +659,7 @@ func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, er
 // not use this on large files as it would cause server to crash.
 func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -666,6 +681,8 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -694,6 +711,8 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 				}
 			}
 			return nil, pathErr
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -715,7 +734,7 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 // semantics are same as io.ReadFull.
 func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (n int64, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -737,6 +756,8 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 	if err != nil {
 		if os.IsNotExist(err) {
 			return 0, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return 0, errFaultyDisk
 		}
 		return 0, err
 	}
@@ -756,6 +777,8 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 			return 0, errFileAccessDenied
 		} else if isSysErrNotDir(err) {
 			return 0, errFileAccessDenied
+		} else if isSysErrIO(err) {
+			return 0, errFaultyDisk
 		}
 		return 0, err
 	}
@@ -807,7 +830,7 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 
 func (s *posix) createFile(volume, path string) (f *os.File, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -829,6 +852,8 @@ func (s *posix) createFile(volume, path string) (f *os.File, err error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -859,6 +884,8 @@ func (s *posix) createFile(volume, path string) (f *os.File, err error) {
 			return nil, errFileAccessDenied
 		} else if os.IsPermission(err) {
 			return nil, errFileAccessDenied
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
 		}
 		return nil, err
 	}
@@ -875,7 +902,7 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 	}
 
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -886,6 +913,9 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 
 	// Validate if disk is indeed free.
 	if err = checkDiskFree(s.diskPath, fileSize); err != nil {
+		if isSysErrIO(err) {
+			return errFaultyDisk
+		}
 		return err
 	}
 
@@ -907,7 +937,7 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 		case isSysErrNoSpace(e):
 			err = errDiskFull
 		case isSysErrIO(e):
-			err = e
+			err = errFaultyDisk
 		default:
 			// For errors: EBADF, EINTR, EINVAL, ENODEV, EPERM, ESPIPE  and ETXTBSY
 			// Appending was failed anyway, returns unexpected error
@@ -922,7 +952,7 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 // path this call explicitly creates it.
 func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -944,7 +974,7 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 // StatFile - get file info.
 func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -979,10 +1009,10 @@ func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
 		// File is really not found.
 		if os.IsNotExist(err) {
 			return FileInfo{}, errFileNotFound
-		}
-
-		// File path cannot be verified since one of the parents is a file.
-		if isSysErrNotDir(err) {
+		} else if isSysErrIO(err) {
+			return FileInfo{}, errFaultyDisk
+		} else if isSysErrNotDir(err) {
+			// File path cannot be verified since one of the parents is a file.
 			return FileInfo{}, errFileNotFound
 		}
 
@@ -1023,6 +1053,8 @@ func deleteFile(basePath, deletePath string) error {
 			return errFileNotFound
 		} else if os.IsPermission(err) {
 			return errFileAccessDenied
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 		return err
 	}
@@ -1041,7 +1073,7 @@ func deleteFile(basePath, deletePath string) error {
 // DeleteFile - delete a file at path.
 func (s *posix) DeleteFile(volume, path string) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -1063,6 +1095,8 @@ func (s *posix) DeleteFile(volume, path string) (err error) {
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 		return err
 	}
@@ -1081,7 +1115,7 @@ func (s *posix) DeleteFile(volume, path string) (err error) {
 // RenameFile - rename source path to destination path atomically.
 func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
 	defer func() {
-		if err == syscall.EIO {
+		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
 		}
 	}()
@@ -1107,6 +1141,8 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 		return err
 	}
@@ -1114,6 +1150,8 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
 		}
 	}
 
@@ -1136,7 +1174,9 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 		// we still need to allow overwriting an empty directory since it represents
 		// an object empty directory.
 		_, err = os.Stat(dstFilePath)
-
+		if isSysErrIO(err) {
+			return errFaultyDisk
+		}
 		if err == nil && !isDirEmpty(dstFilePath) {
 			return errFileAccessDenied
 		}
@@ -1146,6 +1186,9 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 	}
 
 	if err = renameAll(srcFilePath, dstFilePath); err != nil {
+		if isSysErrIO(err) {
+			return errFaultyDisk
+		}
 		return err
 	}
 

cmd/prepare-storage.go

@@ -186,6 +186,23 @@ func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, dr
 		return nil, err
 	}
 
+	// Get the deploymentID if set.
+	format.ID, err = formatXLGetDeploymentID(format, formatConfigs)
+	if err != nil {
+		return nil, err
+	}
+
+	if format.ID == "" {
+		if err = formatXLFixDeploymentID(context.Background(), storageDisks, format); err != nil {
+			return nil, err
+		}
+	}
+
+	logger.SetDeploymentID(format.ID)
+
+	if err = formatXLFixLocalDeploymentID(context.Background(), storageDisks, format); err != nil {
+		return nil, err
+	}
 	return format, nil
 }
 

cmd/routers.go

@@ -47,6 +47,8 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 
 // List of some generic handlers which are applied for all incoming requests.
 var globalHandlers = []HandlerFunc{
+	// set x-amz-request-id header.
+	addrequestIDHeader,
 	// set HTTP security headers such as Content-Security-Policy.
 	addSecurityHeaders,
 	// Forward path style requests to actual host in a bucket federated setup.

cmd/server-main.go

@@ -72,9 +72,6 @@ ENVIRONMENT VARIABLES:
      MINIO_ACCESS_KEY: Custom username or access key of minimum 3 characters in length.
      MINIO_SECRET_KEY: Custom password or secret key of minimum 8 characters in length.
 
-  ENDPOINTS:
-     MINIO_ENDPOINTS: List of all endpoints delimited by ' '.
-
   BROWSER:
      MINIO_BROWSER: To disable web browser access, set this value to "off".
 
@@ -106,9 +103,8 @@ EXAMPLES:
      $ export MINIO_DOMAIN=mydomain.com
      $ {{.HelpName}} --address mydomain.com:9000 /mnt/export
 
-  4. Start minio server on 64 disks server with endpoints through environment variable.
-     $ export MINIO_ENDPOINTS=/mnt/export{1...64}
-     $ {{.HelpName}}
+  4. Start erasure coded minio server on a node with 64 drives.
+     $ {{.HelpName}} /mnt/export{1...64}
 
   5. Start distributed minio server on an 8 node setup with 8 drives each. Run following command on all the 8 nodes.
      $ export MINIO_ACCESS_KEY=minio
@@ -221,6 +217,9 @@ func serverMain(ctx *cli.Context) {
 	// Initialize server config.
 	initConfig()
 
+	// Load logger subsystem
+	loadLoggers()
+
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
@@ -271,15 +270,6 @@ func serverMain(ctx *cli.Context) {
 		logger.Fatal(uiErrUnexpectedError(err), "Unable to configure one of server's RPC services")
 	}
 
-	// Create new notification system.
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
-	if err != nil {
-		logger.Fatal(err, "Unable to initialize the notification system")
-	}
-
-	// Create new policy system.
-	globalPolicySys = NewPolicySys()
-
 	// Initialize Admin Peers inter-node communication only in distributed setup.
 	initGlobalAdminPeers(globalEndpoints)
 
@@ -315,6 +305,25 @@ func serverMain(ctx *cli.Context) {
 		initFederatorBackend(newObject)
 	}
 
+	// Re-enable logging
+	logger.Disable = false
+
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
+
+	// Initialize policy system.
+	if err := globalPolicySys.Init(newObjectLayerFn()); err != nil {
+		logger.Fatal(err, "Unable to initialize policy system")
+	}
+
+	// Create new notification system.
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+
+	// Initialize notification system.
+	if err := globalNotificationSys.Init(newObjectLayerFn()); err != nil {
+		logger.Fatal(err, "Unable to initialize notification system")
+	}
+
 	// Prints the formatted startup message once object layer is initialized.
 	apiEndpoints := getAPIEndpoints(globalMinioAddr)
 	printStartupMessage(apiEndpoints)
@@ -322,9 +331,6 @@ func serverMain(ctx *cli.Context) {
 	// Set uptime time after object layer has initialized.
 	globalBootTime = UTCNow()
 
-	// Re-enable logging
-	logger.Disable = false
-
 	handleSignals()
 }
 

cmd/signature-v4.go

@@ -249,6 +249,12 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 
 	// Save other headers available in the request parameters.
 	for k, v := range req.URL.Query() {
+
+		// Handle the metadata in presigned put query string
+		if strings.Contains(strings.ToLower(k), "x-amz-meta-") {
+			query.Set(k, v[0])
+		}
+
 		if strings.HasPrefix(strings.ToLower(k), "x-amz") {
 			continue
 		}

cmd/storage-class.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"strconv"
@@ -196,18 +197,12 @@ func getRedundancyCount(sc string, totalDisks int) (data, parity int) {
 // Returns per object readQuorum and writeQuorum
 // readQuorum is the minimum required disks to read data.
 // writeQuorum is the minimum required disks to write data.
-func objectQuorumFromMeta(xl xlObjects, partsMetaData []xlMetaV1, errs []error) (objectReadQuorum, objectWriteQuorum int, err error) {
-
+func objectQuorumFromMeta(ctx context.Context, xl xlObjects, partsMetaData []xlMetaV1, errs []error) (objectReadQuorum, objectWriteQuorum int, err error) {
 	// get the latest updated Metadata and a count of all the latest updated xlMeta(s)
-	latestXLMeta, count := getLatestXLMeta(partsMetaData, errs)
+	latestXLMeta, err := getLatestXLMeta(ctx, partsMetaData, errs)
 
-	// latestXLMeta is updated most recently.
-	// We implicitly assume that all the xlMeta(s) have same dataBlocks and parityBlocks.
-	// We now check that at least dataBlocks number of xlMeta is available. This means count
-	// should be greater than or equal to dataBlocks field of latestXLMeta. If not we throw read quorum error.
-	if count < latestXLMeta.Erasure.DataBlocks {
-		// This is the case when we can't reliably deduce object quorum
-		return 0, 0, errXLReadQuorum
+	if err != nil {
+		return 0, 0, err
 	}
 
 	// Since all the valid erasure code meta updated at the same time are equivalent, pass dataBlocks

cmd/storage-class_test.go

@@ -303,7 +303,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		{parts7, errs7, 14, 15, nil},
 	}
 	for i, tt := range tests {
-		actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(*xl, tt.parts, tt.errs)
+		actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(context.Background(), *xl, tt.parts, tt.errs)
 		if tt.expectedError != nil && err == nil {
 			t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
 			return

cmd/test-utils_test.go

@@ -353,10 +353,8 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
 	globalMinioHost = host
 	globalMinioPort = port
 	globalMinioAddr = getEndpointsLocalAddr(testServer.Disks)
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, testServer.Disks)
-	if err != nil {
-		t.Fatalf("Unable to create new notification system. %v", err)
-	}
+
+	globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks)
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
@@ -1720,9 +1718,7 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro
 	}
 
 	// Create new notification system.
-	if globalNotificationSys, err = NewNotificationSys(globalServerConfig, endpoints); err != nil {
-		return nil, err
-	}
+	globalNotificationSys = NewNotificationSys(globalServerConfig, endpoints)
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
@@ -1859,7 +1855,7 @@ func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketN
 	}
 
 	// expected error response in bytes when objectLayer is not initialized, or set to `nil`.
-	expectedErrResponse := encodeResponse(getAPIErrorResponse(getAPIError(ErrAccessDenied), getGetObjectURL("", bucketName, objectName)))
+	expectedErrResponse := encodeResponse(getAPIErrorResponse(getAPIError(ErrAccessDenied), getGetObjectURL("", bucketName, objectName), ""))
 
 	// HEAD HTTTP request doesn't contain response body.
 	if anonReq.Method != "HEAD" {
@@ -1962,7 +1958,7 @@ func ExecObjectLayerAPINilTest(t TestErrHandler, bucketName, objectName, instanc
 	}
 	// expected error response in bytes when objectLayer is not initialized, or set to `nil`.
 	expectedErrResponse := encodeResponse(getAPIErrorResponse(getAPIError(ErrServerNotInitialized),
-		getGetObjectURL("", bucketName, objectName)))
+		getGetObjectURL("", bucketName, objectName), ""))
 
 	// HEAD HTTP Request doesn't contain body in its response,
 	// for other type of HTTP requests compare the response body content with the expected one.

cmd/tree-walk.go

@@ -86,7 +86,7 @@ func filterMatchingPrefix(entries []string, prefixEntry string) []string {
 }
 
 // "listDir" function of type listDirFunc returned by listDirFactory() - explained below.
-type listDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool, err error)
+type listDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool)
 
 // A function isLeaf of type isLeafFunc is used to detect if an entry is a leaf entry. There are four scenarios
 // where isLeaf should behave differently:
@@ -141,16 +141,8 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 			markerBase = markerSplit[1]
 		}
 	}
-	entries, delayIsLeaf, err := listDir(bucket, prefixDir, entryPrefixMatch)
-	if err != nil {
-		select {
-		case <-endWalkCh:
-			return errWalkAbort
-		case resultCh <- treeWalkResult{err: err}:
-			return err
-		}
-	}
 
+	entries, delayIsLeaf := listDir(bucket, prefixDir, entryPrefixMatch)
 	// When isleaf check is delayed, make sure that it is set correctly here.
 	if delayIsLeaf && isLeaf == nil {
 		return errInvalidArgument

cmd/tree-walk_test.go

@@ -196,7 +196,7 @@ func TestTreeWalk(t *testing.T) {
 		return len(entries) == 0
 	}
 
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk)
+	listDir := listDirFactory(context.Background(), isLeaf, disk)
 	// Simple test for prefix based walk.
 	testTreeWalkPrefix(t, listDir, isLeaf, isLeafDir)
 	// Simple test when marker is set.
@@ -240,7 +240,7 @@ func TestTreeWalkTimeout(t *testing.T) {
 		return len(entries) == 0
 	}
 
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk)
+	listDir := listDirFactory(context.Background(), isLeaf, disk)
 
 	// TreeWalk pool with 2 seconds timeout for tree-walk go routines.
 	pool := newTreeWalkPool(2 * time.Second)
@@ -315,7 +315,7 @@ func TestListDir(t *testing.T) {
 	// create listDir function.
 	listDir := listDirFactory(context.Background(), func(volume, prefix string) bool {
 		return !hasSuffix(prefix, slashSeparator)
-	}, xlTreeWalkIgnoredErrs, disk1, disk2)
+	}, disk1, disk2)
 
 	// Create file1 in fsDir1 and file2 in fsDir2.
 	disks := []StorageAPI{disk1, disk2}
@@ -327,10 +327,7 @@ func TestListDir(t *testing.T) {
 	}
 
 	// Should list "file1" from fsDir1.
-	entries, _, err := listDir(volume, "", "")
-	if err != nil {
-		t.Error(err)
-	}
+	entries, _ := listDir(volume, "", "")
 	if len(entries) != 2 {
 		t.Fatal("Expected the number of entries to be 2")
 	}
@@ -348,10 +345,7 @@ func TestListDir(t *testing.T) {
 	}
 
 	// Should list "file2" from fsDir2.
-	entries, _, err = listDir(volume, "", "")
-	if err != nil {
-		t.Error(err)
-	}
+	entries, _ = listDir(volume, "", "")
 	if len(entries) != 1 {
 		t.Fatal("Expected the number of entries to be 1")
 	}
@@ -362,13 +356,6 @@ func TestListDir(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	// None of the disks are available, should get
-	// errDiskNotFound. Since errDiskNotFound is an ignored error,
-	// we should get nil.
-	_, _, err = listDir(volume, "", "")
-	if err != nil {
-		t.Errorf("expected nil error but found %v.", err)
-	}
 }
 
 // TestRecursiveWalk - tests if treeWalk returns entries correctly with and
@@ -400,7 +387,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
 	}
 
 	// Create listDir function.
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1)
+	listDir := listDirFactory(context.Background(), isLeaf, disk1)
 
 	// Create the namespace.
 	var files = []string{
@@ -515,7 +502,7 @@ func TestSortedness(t *testing.T) {
 	}
 
 	// Create listDir function.
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1)
+	listDir := listDirFactory(context.Background(), isLeaf, disk1)
 
 	// Create the namespace.
 	var files = []string{
@@ -598,7 +585,7 @@ func TestTreeWalkIsEnd(t *testing.T) {
 	}
 
 	// Create listDir function.
-	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1)
+	listDir := listDirFactory(context.Background(), isLeaf, disk1)
 
 	// Create the namespace.
 	var files = []string{

cmd/ui-errors.go

@@ -19,8 +19,8 @@ package cmd
 var (
 	uiErrInvalidConfig = newUIErrFn(
 		"Invalid value found in the configuration file",
-		"Please ensure a valid value in the configuration file, for more details refer https://docs.minio.io/docs/minio-server-configuration-guide",
-		"",
+		"Please ensure a valid value in the configuration file",
+		"For more details, refer to https://docs.minio.io/docs/minio-server-configuration-guide",
 	)
 
 	uiErrInvalidBrowserValue = newUIErrFn(
@@ -120,12 +120,6 @@ Example 1:
 		"",
 	)
 
-	uiErrUnableToReadFromBackend = newUIErrFn(
-		"Unable to read from the backend",
-		"Please ensure Minio binary has read permissions for the backend",
-		"",
-	)
-
 	uiErrPortAlreadyInUse = newUIErrFn(
 		"Port is already in use",
 		"Please ensure no other program uses the same address/port",

cmd/update-main.go

@@ -456,32 +456,32 @@ func getUpdateInfo(timeout time.Duration, mode string) (updateMsg string, sha256
 	return prepareUpdateMessage(downloadURL, older), sha256Hex, currentReleaseTime, latestReleaseTime, nil
 }
 
-func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (successMsg string, err error) {
+func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (updateStatusMsg string, err error) {
 	if !ok {
-		successMsg = greenColorSprintf("Minio update to version RELEASE.%s canceled.",
+		updateStatusMsg = greenColorSprintf("Minio update to version RELEASE.%s canceled.",
 			latestReleaseTime.Format(minioReleaseTagTimeLayout))
-		return successMsg, nil
+		return updateStatusMsg, nil
 	}
 	var sha256Sum []byte
 	sha256Sum, err = hex.DecodeString(sha256Hex)
 	if err != nil {
-		return successMsg, err
+		return updateStatusMsg, err
 	}
 
 	resp, err := http.Get(getDownloadURL(releaseTimeToReleaseTag(latestReleaseTime)))
 	if err != nil {
-		return successMsg, err
+		return updateStatusMsg, err
 	}
 	defer resp.Body.Close()
 
 	// FIXME: add support for gpg verification as well.
-	if err = update.RollbackError(update.Apply(resp.Body,
+	if err = update.Apply(resp.Body,
 		update.Options{
 			Hash:     crypto.SHA256,
 			Checksum: sha256Sum,
 		},
-	)); err != nil {
-		return successMsg, err
+	); err != nil {
+		return updateStatusMsg, err
 	}
 
 	return greenColorSprintf("Minio updated to version RELEASE.%s successfully.",
@@ -497,6 +497,7 @@ func shouldUpdate(quiet bool, sha256Hex string, latestReleaseTime time.Time) (ok
 }
 
 var greenColorSprintf = color.New(color.FgGreen, color.Bold).SprintfFunc()
+var redColorSprintf = color.New(color.FgRed, color.Bold).SprintfFunc()
 
 func mainUpdate(ctx *cli.Context) {
 	if len(ctx.Args()) != 0 {
@@ -527,13 +528,14 @@ func mainUpdate(ctx *cli.Context) {
 	// if the in-place update is disabled then we shouldn't ask the
 	// user to update the binaries.
 	if strings.Contains(updateMsg, minioReleaseURL) && !globalInplaceUpdateDisabled {
-		var successMsg string
-		successMsg, err = doUpdate(sha256Hex, latestReleaseTime, shouldUpdate(quiet, sha256Hex, latestReleaseTime))
+		var updateStatusMsg string
+		updateStatusMsg, err = doUpdate(sha256Hex, latestReleaseTime, shouldUpdate(quiet, sha256Hex, latestReleaseTime))
 		if err != nil {
+			logger.Info(redColorSprintf("Unable to update ‘minio’."))
 			logger.Info(err.Error())
 			os.Exit(-1)
 		}
-		logger.Info(successMsg)
+		logger.Info(updateStatusMsg)
 		os.Exit(1)
 	}
 }

cmd/utils.go

@@ -142,6 +142,10 @@ const (
 	// Maximum Part ID for multipart upload is 10000
 	// (Acceptable values range from 1 to 10000 inclusive)
 	globalMaxPartID = 10000
+
+	// Default values used while communicating with the cloud backends
+	defaultDialTimeout   = 30 * time.Second
+	defaultDialKeepAlive = 30 * time.Second
 )
 
 // isMaxObjectSize - verify if max object size
@@ -263,8 +267,8 @@ func NewCustomHTTPTransport() *http.Transport {
 	return &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
 		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
+			Timeout:   defaultDialTimeout,
+			KeepAlive: defaultDialKeepAlive,
 		}).DialContext,
 		MaxIdleConns:          1024,
 		MaxIdleConnsPerHost:   1024,
@@ -327,7 +331,7 @@ func ceilFrac(numerator, denominator int64) (ceil int64) {
 }
 
 // Returns context with ReqInfo details set in the context.
-func newContext(r *http.Request, api string) context.Context {
+func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -337,6 +341,7 @@ func newContext(r *http.Request, api string) context.Context {
 		object = prefix
 	}
 	reqInfo := &logger.ReqInfo{
+		RequestID:  w.Header().Get(responseRequestIDKey),
 		RemoteHost: handlers.GetSourceIP(r),
 		UserAgent:  r.Header.Get("user-agent"),
 		API:        api,

cmd/utils_test.go

@@ -445,6 +445,7 @@ func TestCeilFrac(t *testing.T) {
 // Test if isErrIgnored works correctly.
 func TestIsErrIgnored(t *testing.T) {
 	var errIgnored = fmt.Errorf("ignored error")
+	ignoredErrs := append(baseIgnoredErrs, errIgnored)
 	var testCases = []struct {
 		err     error
 		ignored bool
@@ -457,9 +458,13 @@ func TestIsErrIgnored(t *testing.T) {
 			err:     errIgnored,
 			ignored: true,
 		},
+		{
+			err:     errFaultyDisk,
+			ignored: true,
+		},
 	}
 	for i, testCase := range testCases {
-		if ok := IsErrIgnored(testCase.err, errIgnored); ok != testCase.ignored {
+		if ok := IsErrIgnored(testCase.err, ignoredErrs...); ok != testCase.ignored {
 			t.Errorf("Test: %d, Expected %t, got %t", i+1, testCase.ignored, ok)
 		}
 	}

cmd/web-handlers.go

@@ -31,7 +31,6 @@ import (
 	"strings"
 	"time"
 
-	etcd "github.com/coreos/etcd/client"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/rpc/v2/json2"
@@ -139,7 +138,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
 
 	if globalDNSConfig != nil {
 		if _, err := globalDNSConfig.Get(args.BucketName); err != nil {
-			if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
+			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(context.Background(), args.BucketName, globalServerConfig.GetRegion()); err != nil {
 					return toJSONError(err)
@@ -629,7 +628,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Extract incoming metadata if any.
-	metadata, err := extractMetadataFromHeader(context.Background(), r.Header)
+	metadata, err := extractMetadata(context.Background(), r)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
@@ -841,6 +840,7 @@ func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolic
 		if _, ok := err.(BucketPolicyNotFound); !ok {
 			return toJSONError(err, args.BucketName)
 		}
+		return err
 	}
 
 	policyInfo, err := PolicyToBucketAccessPolicy(bucketPolicy)

cmd/xl-sets.go

@@ -263,16 +263,6 @@ func newXLSets(endpoints EndpointList, format *formatXLV3, setCount int, drivesP
 	// Connect disks right away.
 	s.connectDisks()
 
-	// Initialize notification system.
-	if err := globalNotificationSys.Init(s); err != nil {
-		return nil, fmt.Errorf("Unable to initialize notification system. %v", err)
-	}
-
-	// Initialize policy system.
-	if err := globalPolicySys.Init(s); err != nil {
-		return nil, fmt.Errorf("Unable to initialize policy system. %v", err)
-	}
-
 	// Start the disk monitoring and connect routine.
 	go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
 
@@ -632,8 +622,8 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir(). Sets passes set of disks.
-func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, treeWalkIgnoredErrs []error, sets ...[]StorageAPI) listDirFunc {
-	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string, err error) {
+func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, sets ...[]StorageAPI) listDirFunc {
+	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string) {
 		for _, disk := range disks {
 			if disk == nil {
 				continue
@@ -641,15 +631,10 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 
 			var entries []string
 			var newEntries []string
+			var err error
 			entries, err = disk.ListDir(bucket, prefixDir, -1)
 			if err != nil {
-				// For any reason disk was deleted or goes offline, continue
-				// and list from other disks if possible.
-				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
-					continue
-				}
-				logger.LogIf(ctx, err)
-				return nil, err
+				continue
 			}
 
 			// Find elements in entries which are not in mergedEntries
@@ -668,18 +653,13 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 				sort.Strings(mergedEntries)
 			}
 		}
-		return mergedEntries, nil
+		return mergedEntries
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
 		for _, disks := range sets {
-			var entries []string
-			entries, err = listDirInternal(bucket, prefixDir, prefixEntry, disks)
-			if err != nil {
-				return nil, false, err
-			}
-
+			entries := listDirInternal(bucket, prefixDir, prefixEntry, disks)
 			var newEntries []string
 			// Find elements in entries which are not in mergedEntries
 			for _, entry := range entries {
@@ -698,7 +678,7 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 			}
 		}
 		mergedEntries, delayIsLeaf = filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
-		return mergedEntries, delayIsLeaf, nil
+		return mergedEntries, delayIsLeaf
 	}
 	return listDir
 }
@@ -741,7 +721,7 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
 			setDisks = append(setDisks, set.getLoadBalancedDisks())
 		}
 
-		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, xlTreeWalkIgnoredErrs, setDisks...)
+		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, setDisks...)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 
@@ -1276,13 +1256,15 @@ func (s *xlSets) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir(). Sets passes set of disks.
 func listDirSetsHealFactory(isLeaf isLeafFunc, sets ...[]StorageAPI) listDirFunc {
-	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string, err error) {
+	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string) {
 		for _, disk := range disks {
 			if disk == nil {
 				continue
 			}
+
 			var entries []string
 			var newEntries []string
+			var err error
 			entries, err = disk.ListDir(bucket, prefixDir, -1)
 			if err != nil {
 				continue
@@ -1315,18 +1297,14 @@ func listDirSetsHealFactory(isLeaf isLeafFunc, sets ...[]StorageAPI) listDirFunc
 				sort.Strings(mergedEntries)
 			}
 		}
-		return mergedEntries, nil
+		return mergedEntries
 
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
 		for _, disks := range sets {
-			var entries []string
-			entries, err = listDirInternal(bucket, prefixDir, prefixEntry, disks)
-			if err != nil {
-				return nil, false, err
-			}
+			entries := listDirInternal(bucket, prefixDir, prefixEntry, disks)
 
 			var newEntries []string
 			// Find elements in entries which are not in mergedEntries
@@ -1345,7 +1323,7 @@ func listDirSetsHealFactory(isLeaf isLeafFunc, sets ...[]StorageAPI) listDirFunc
 				sort.Strings(mergedEntries)
 			}
 		}
-		return mergedEntries, false, nil
+		return mergedEntries, false
 	}
 	return listDir
 }
@@ -1455,24 +1433,3 @@ func (s *xlSets) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, de
 	// Return error at the end.
 	return loi, toObjectErr(err, bucket, prefix)
 }
-
-// ListLocks from all sets, aggregate them and return.
-func (s *xlSets) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) (lockInfo []VolumeLockInfo, err error) {
-	for _, set := range s.sets {
-		var setLockInfo []VolumeLockInfo
-		setLockInfo, err = set.ListLocks(ctx, bucket, prefix, duration)
-		if err != nil {
-			return nil, err
-		}
-		lockInfo = append(lockInfo, setLockInfo...)
-	}
-	return lockInfo, nil
-}
-
-// Clear all requested locks on all sets.
-func (s *xlSets) ClearLocks(ctx context.Context, lockInfo []VolumeLockInfo) error {
-	for _, set := range s.sets {
-		set.ClearLocks(ctx, lockInfo)
-	}
-	return nil
-}

cmd/xl-v1-common.go

@@ -51,6 +51,8 @@ func (xl xlObjects) parentDirIsObject(ctx context.Context, bucket, parent string
 	return isParentDirObject(parent)
 }
 
+var xlTreeWalkIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errVolumeNotFound, errFileNotFound)
+
 // isObject - returns `true` if the prefix is an object i.e if
 // `xl.json` exists at the leaf, false otherwise.
 func (xl xlObjects) isObject(bucket, prefix string) (ok bool) {

cmd/xl-v1-healing-common.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
@@ -118,8 +119,14 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []xlMetaV1, errs []error)
 	return onlineDisks, modTime
 }
 
-// Returns one of the latest updated xlMeta files and count of total valid xlMeta(s) updated latest
-func getLatestXLMeta(partsMetadata []xlMetaV1, errs []error) (xlMetaV1, int) {
+// Returns the latest updated xlMeta files and error in case of failure.
+func getLatestXLMeta(ctx context.Context, partsMetadata []xlMetaV1, errs []error) (xlMetaV1, error) {
+
+	// There should be atleast half correct entries, if not return failure
+	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, globalXLSetDriveCount/2); reducedErr != nil {
+		return xlMetaV1{}, reducedErr
+	}
+
 	// List all the file commit ids from parts metadata.
 	modTimes := listObjectModtimes(partsMetadata, errs)
 
@@ -137,8 +144,11 @@ func getLatestXLMeta(partsMetadata []xlMetaV1, errs []error) (xlMetaV1, int) {
 			count++
 		}
 	}
-	// Return one of the latest xlMetaData, and the count of lastest updated xlMeta files
-	return latestXLMeta, count
+	if count < len(partsMetadata)/2 {
+		return xlMetaV1{}, errXLReadQuorum
+	}
+
+	return latestXLMeta, nil
 }
 
 // disksWithAllParts - This function needs to be called with
@@ -175,7 +185,10 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			// buffer is passed
 			_, hErr := onlineDisk.ReadFile(bucket, partPath, 0, buffer, verifier)
 
-			_, isCorrupt := hErr.(hashMismatchError)
+			isCorrupt := false
+			if hErr != nil {
+				isCorrupt = strings.HasPrefix(hErr.Error(), "Bitrot verification mismatch - expected ")
+			}
 			switch {
 			case isCorrupt:
 				fallthrough

cmd/xl-v1-healing.go

@@ -611,7 +611,10 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 	// Read metadata files from all the disks
 	partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object)
 
-	latestXLMeta, _ := getLatestXLMeta(partsMetadata, errs)
+	latestXLMeta, err := getLatestXLMeta(ctx, partsMetadata, errs)
+	if err != nil {
+		return hr, toObjectErr(err, bucket, object)
+	}
 
 	// Lock the object before healing.
 	objectLock := xl.nsMutex.NewNSLock(bucket, object)

cmd/xl-v1-healing_test.go

@@ -145,6 +145,6 @@ func TestHealObjectXL(t *testing.T) {
 	_, err = obj.HealObject(context.Background(), bucket, object, false)
 	// since majority of xl.jsons are not available, object quorum can't be read properly and error will be errXLReadQuorum
 	if _, ok := err.(InsufficientReadQuorum); !ok {
-		t.Errorf("Expected %v but received %v", InsufficientWriteQuorum{}, err)
+		t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err)
 	}
 }

cmd/xl-v1-list-objects.go

@@ -24,23 +24,19 @@ import (
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir()
-func listDirFactory(ctx context.Context, isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks ...StorageAPI) listDirFunc {
+func listDirFactory(ctx context.Context, isLeaf isLeafFunc, disks ...StorageAPI) listDirFunc {
 	// Returns sorted merged entries from all the disks.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool, err error) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
 		for _, disk := range disks {
 			if disk == nil {
 				continue
 			}
 			var entries []string
 			var newEntries []string
+			var err error
 			entries, err = disk.ListDir(bucket, prefixDir, -1)
 			if err != nil {
-				// For any reason disk was deleted or goes offline, continue
-				// and list from other disks if possible.
-				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
-					continue
-				}
-				return nil, false, err
+				continue
 			}
 
 			// Find elements in entries which are not in mergedEntries
@@ -60,7 +56,7 @@ func listDirFactory(ctx context.Context, isLeaf isLeafFunc, treeWalkIgnoredErrs
 			}
 		}
 		mergedEntries, delayIsLeaf = filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
-		return mergedEntries, delayIsLeaf, nil
+		return mergedEntries, delayIsLeaf
 	}
 	return listDir
 }
@@ -79,7 +75,7 @@ func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, del
 		endWalkCh = make(chan struct{})
 		isLeaf := xl.isObject
 		isLeafDir := xl.isObjectDir
-		listDir := listDirFactory(ctx, isLeaf, xlTreeWalkIgnoredErrs, xl.getLoadBalancedDisks()...)
+		listDir := listDirFactory(ctx, isLeaf, xl.getLoadBalancedDisks()...)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 

cmd/xl-v1-multipart.go

@@ -330,7 +330,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 		uploadIDPath)
 
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(xl, partsMetadata, errs)
+	_, writeQuorum, err := objectQuorumFromMeta(ctx, xl, partsMetadata, errs)
 	if err != nil {
 		return pi, toObjectErr(err, bucket, object)
 	}
@@ -612,7 +612,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), minioMetaMultipartBucket, uploadIDPath)
 
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(xl, partsMetadata, errs)
+	_, writeQuorum, err := objectQuorumFromMeta(ctx, xl, partsMetadata, errs)
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
 	}
@@ -829,7 +829,7 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up
 	partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), minioMetaMultipartBucket, uploadIDPath)
 
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(xl, partsMetadata, errs)
+	_, writeQuorum, err := objectQuorumFromMeta(ctx, xl, partsMetadata, errs)
 	if err != nil {
 		return toObjectErr(err, bucket, object)
 	}

cmd/xl-v1-object.go

@@ -89,7 +89,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc
 	metaArr, errs := readAllXLMetadata(ctx, storageDisks, srcBucket, srcObject)
 
 	// get Quorum for this object
-	readQuorum, writeQuorum, err := objectQuorumFromMeta(xl, metaArr, errs)
+	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, xl, metaArr, errs)
 	if err != nil {
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
@@ -208,7 +208,7 @@ func (xl xlObjects) getObject(ctx context.Context, bucket, object string, startO
 	metaArr, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object)
 
 	// get Quorum for this object
-	readQuorum, _, err := objectQuorumFromMeta(xl, metaArr, errs)
+	readQuorum, _, err := objectQuorumFromMeta(ctx, xl, metaArr, errs)
 	if err != nil {
 		return toObjectErr(err, bucket, object)
 	}
@@ -382,7 +382,7 @@ func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (o
 	metaArr, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object)
 
 	// get Quorum for this object
-	readQuorum, _, err := objectQuorumFromMeta(xl, metaArr, errs)
+	readQuorum, _, err := objectQuorumFromMeta(ctx, xl, metaArr, errs)
 	if err != nil {
 		return objInfo, err
 	}
@@ -771,7 +771,7 @@ func (xl xlObjects) deleteObject(ctx context.Context, bucket, object string) err
 		// Read metadata associated with the object from all disks.
 		metaArr, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object)
 		// get Quorum for this object
-		_, writeQuorum, err = objectQuorumFromMeta(xl, metaArr, errs)
+		_, writeQuorum, err = objectQuorumFromMeta(ctx, xl, metaArr, errs)
 		if err != nil {
 			return err
 		}

cmd/xl-v1.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"context"
 	"sort"
-	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
@@ -49,9 +48,6 @@ type xlObjects struct {
 	listPool *treeWalkPool
 }
 
-// list of all errors that can be ignored in tree walk operation in XL
-var xlTreeWalkIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errVolumeNotFound, errFileNotFound)
-
 // Shutdown function for object storage interface.
 func (xl xlObjects) Shutdown(ctx context.Context) error {
 	// Add any object layer shutdown activities here.
@@ -59,63 +55,6 @@ func (xl xlObjects) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-// Locking operations
-
-// List namespace locks held in object layer
-func (xl xlObjects) ListLocks(ctx context.Context, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	xl.nsMutex.lockMapMutex.Lock()
-	defer xl.nsMutex.lockMapMutex.Unlock()
-	// Fetch current time once instead of fetching system time for every lock.
-	timeNow := UTCNow()
-	volumeLocks := []VolumeLockInfo{}
-
-	for param, debugLock := range xl.nsMutex.debugLockMap {
-		if param.volume != bucket {
-			continue
-		}
-		// N B empty prefix matches all param.path.
-		if !hasPrefix(param.path, prefix) {
-			continue
-		}
-
-		volLockInfo := VolumeLockInfo{
-			Bucket:                param.volume,
-			Object:                param.path,
-			LocksOnObject:         debugLock.counters.total,
-			TotalBlockedLocks:     debugLock.counters.blocked,
-			LocksAcquiredOnObject: debugLock.counters.granted,
-		}
-		// Filter locks that are held on bucket, prefix.
-		for opsID, lockInfo := range debugLock.lockInfo {
-			// filter locks that were held for longer than duration.
-			elapsed := timeNow.Sub(lockInfo.since)
-			if elapsed < duration {
-				continue
-			}
-			// Add locks that are held for longer than duration.
-			volLockInfo.LockDetailsOnObject = append(volLockInfo.LockDetailsOnObject,
-				OpsLockState{
-					OperationID: opsID,
-					LockSource:  lockInfo.lockSource,
-					LockType:    lockInfo.lType,
-					Status:      lockInfo.status,
-					Since:       lockInfo.since,
-				})
-			volumeLocks = append(volumeLocks, volLockInfo)
-		}
-	}
-	return volumeLocks, nil
-}
-
-// Clear namespace locks held in object layer
-func (xl xlObjects) ClearLocks(ctx context.Context, volLocks []VolumeLockInfo) error {
-	// Remove lock matching bucket/prefix held longer than duration.
-	for _, volLock := range volLocks {
-		xl.nsMutex.ForceUnlock(volLock.Bucket, volLock.Object)
-	}
-	return nil
-}
-
 // byDiskTotal is a collection satisfying sort.Interface.
 type byDiskTotal []DiskInfo
 

docs/distributed/README.md

@@ -43,13 +43,13 @@ To start a distributed Minio instance, you just need to pass drive locations as
 *Note* 
 
 - All the nodes running distributed Minio need to have same access key and secret key for the nodes to connect. To achieve this, you need to export access key and secret key as environment variables on all the nodes before executing Minio server command.
-- Disks used for Minio distributed should be fresh with no pre-existing data. 
+- Minio distributed mode requires fresh directories. If required, the drives can be shared with other applications. You can do this by using a sub-directory exclusive to minio. For example, if you have mounted your volume under `/export`, pass `/export/data` as arguments to Minio server.
 - The IP addresses and drive paths below are for demonstration purposes only, you need to replace these with the actual IP addresses and drive paths/folders.
 - Servers running distributed Minio instances should be less than 3 seconds apart. You can use [NTP](http://www.ntp.org/) as a best practice to ensure consistent times across servers. 
 - Running Distributed Minio on Windows is experimental as of now. Please proceed with caution. 
 
-Example 1: Start distributed Minio instance with 1 drive each on 8 nodes, by running this command on all the 8 nodes.
-
+Example 1: Start distributed Minio instance on 8 nodes with 1 drive each (pictured below), by running this command on all the 8 nodes:
+![Distributed Minio, 8 nodes with 1 disk each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_8.jpg?raw=true)
 #### GNU/Linux and macOS
 
 ```shell
@@ -60,6 +60,7 @@ minio server http://192.168.1.11/export1 http://192.168.1.12/export2 \
                http://192.168.1.15/export5 http://192.168.1.16/export6 \
                http://192.168.1.17/export7 http://192.168.1.18/export8
 ```
+
 #### Windows 
 
 ```cmd
@@ -71,23 +72,30 @@ minio.exe server http://192.168.1.11/C:/data http://192.168.1.12/C:/data ^
                   http://192.168.1.17/C:/data http://192.168.1.18/C:/data
 ```
 
-![Distributed Minio, 8 nodes with 1 disk each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_8.jpg?raw=true)
 
-Example 2: Start distributed Minio instance with 4 drives each on 4 nodes, by running this command on all the 4 nodes.
+Example 2: Start distributed Minio instance on 4 nodes with 4 drives (pictured below), by running this command on all the 4 nodes:
+![Distributed Minio, 4 nodes with 4 disks each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_16.jpg?raw=true)
 
 #### GNU/Linux and macOS
 
 ```shell
 export MINIO_ACCESS_KEY=<ACCESS_KEY>
 export MINIO_SECRET_KEY=<SECRET_KEY>
-minio server http://192.168.1.11/export1 http://192.168.1.11/export2 \
-               http://192.168.1.11/export3 http://192.168.1.11/export4 \
-               http://192.168.1.12/export1 http://192.168.1.12/export2 \
-               http://192.168.1.12/export3 http://192.168.1.12/export4 \
-               http://192.168.1.13/export1 http://192.168.1.13/export2 \
-               http://192.168.1.13/export3 http://192.168.1.13/export4 \
-               http://192.168.1.14/export1 http://192.168.1.14/export2 \
-               http://192.168.1.14/export3 http://192.168.1.14/export4
+minio server http://192.168.1.11/export1 http://192.168.1.12/export1 \
+               http://192.168.1.13/export1 http://192.168.1.14/export1 \
+               http://192.168.1.11/export2 http://192.168.1.12/export2 \
+               http://192.168.1.13/export2 http://192.168.1.14/export2 \
+               http://192.168.1.11/export3 http://192.168.1.12/export3 \
+               http://192.168.1.13/export3 http://192.168.1.14/export3 \
+               http://192.168.1.11/export4 http://192.168.1.12/export4 \
+               http://192.168.1.13/export4 http://192.168.1.14/export4
+```
+
+##### Short version of the same command:
+```shell
+export MINIO_ACCESS_KEY=<ACCESS_KEY>
+export MINIO_SECRET_KEY=<SECRET_KEY>
+minio server http://192.168.1.{11...14}/export{1...4}
 ```
 
 #### Windows
@@ -95,17 +103,24 @@ minio server http://192.168.1.11/export1 http://192.168.1.11/export2 \
 ```cmd
 set MINIO_ACCESS_KEY=<ACCESS_KEY>
 set MINIO_SECRET_KEY=<SECRET_KEY>
-minio.exe server http://192.168.1.11/C:/data1 http://192.168.1.11/C:/data2 ^
-                  http://192.168.1.11/C:/data3 http://192.168.1.11/C:/data4 ^
-                  http://192.168.1.12/C:/data1 http://192.168.1.12/C:/data2 ^
-                  http://192.168.1.12/C:/data3 http://192.168.1.12/C:/data4 ^
-                  http://192.168.1.13/C:/data1 http://192.168.1.13/C:/data2 ^
-                  http://192.168.1.13/C:/data3 http://192.168.1.13/C:/data4 ^                  
-                  http://192.168.1.14/C:/data1 http://192.168.1.14/C:/data2 ^
-                  http://192.168.1.14/C:/data3 http://192.168.1.14/C:/data4
+minio.exe server http://192.168.1.11/C:/data1 http://192.168.1.12/C:/data1 ^
+                  http://192.168.1.13/C:/data1 http://192.168.1.14/C:/data1 ^
+                  http://192.168.1.11/C:/data2 http://192.168.1.12/C:/data2 ^
+                  http://192.168.1.13/C:/data2 http://192.168.1.14/C:/data2 ^
+                  http://192.168.1.11/C:/data3 http://192.168.1.12/C:/data3 ^
+                  http://192.168.1.13/C:/data3 http://192.168.1.14/C:/data3 ^
+                  http://192.168.1.11/C:/data4 http://192.168.1.12/C:/data4 ^
+                  http://192.168.1.13/C:/data4 http://192.168.1.14/C:/data4
 ```
 
-![Distributed Minio, 4 nodes with 4 disks each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_16.jpg?raw=true)
+##### Short version of the same command:
+```cmd
+set MINIO_ACCESS_KEY=<ACCESS_KEY>
+set MINIO_SECRET_KEY=<SECRET_KEY>
+minio.exe server http://192.168.1.{1...14}/C:/data{1...4}
+```
+__NOTE:__ `{1...n}` shown in shortened examples above have 3 dots! Using only 2 dots `{1..4}` will be interpreted by your shell and won't be passed to minio server, affecting the erasure coding order, which may impact performance and high availability. __Always use `{1...n}` (3 dots!) to allow minio server to optimally stripe erasure-coded data__
+
 
 ## 3. Test your setup
 

docs/federation/lookup/README.md

@@ -10,7 +10,7 @@ Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quicks
 Bucket lookup from DNS federation requires two dependencies
 
 - etcd (for config, bucket SRV records)
-- coredns (for DNS management based on populated bucket SRV records, optional)
+- CoreDNS (for DNS management based on populated bucket SRV records, optional)
 
 ## Architecture
 
@@ -73,10 +73,22 @@ NOTE: `mybucket` only exists on one cluster either `cluster1` or `cluster2` this
 is decided by how `domain.com` gets resolved, if there is a round-robin DNS on `domain.com` then
 it is randomized which cluster might provision the bucket.
 
-### 3. Test your setup
+### 3. Upgrading to `etcdv3` API
+
+Users running Minio federation from release `RELEASE.2018-06-09T03-43-35Z` to `RELEASE.2018-07-10T01-42-11Z`, should migrate the existing bucket data on etcd server to `etcdv3` API, and update CoreDNS version to `1.2.0` before updating their Minio server to the latest version.
+
+Here is some background on why this is needed - Minio server release `RELEASE.2018-06-09T03-43-35Z` to `RELEASE.2018-07-10T01-42-11Z` used etcdv2 API to store bucket data to etcd server. This was due to `etcdv3` support not available for CoreDNS server. So, even if Minio used `etcdv3` API to store bucket data, CoreDNS wouldn't be able to read and serve it as DNS records.
+
+Now that CoreDNS [supports etcdv3](https://coredns.io/2018/07/11/coredns-1.2.0-release/), Minio server uses `etcdv3` API to store bucket data to etcd server. As `etcdv2` and `etcdv3` APIs are not compatible, data stored using `etcdv2` API is not visible to the `etcdv3` API. So, bucket data stored by previous Minio version will not be visible to current Minio version, until a migration is done.
+
+CoreOS team has documented the steps required to migrate existing data from `etcdv2` to `etcdv3` in [this blog post](https://coreos.com/blog/migrating-applications-etcd-v3.html). Please refer the post and migrate etcd data to `etcdv3` API.
+
+### 4. Test your setup
+
 To test this setup, access the Minio server via browser or [`mc`](https://docs.minio.io/docs/minio-client-quickstart-guide). You’ll see the uploaded files are accessible from the all the Minio endpoints.
 
 # Explore Further
+
 - [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
 - [Use `aws-cli` with Minio Server](https://docs.minio.io/docs/aws-cli-with-minio)
 - [Use `s3cmd` with Minio Server](https://docs.minio.io/docs/s3cmd-with-minio)