Performance improvements to SELECT API on certain query operations (#6752)

This improves the performance of certain queries dramatically,
such as 'count(*)' etc.

Without this PR
```
~ time mc select --query "select count(*) from S3Object" myminio/sjm-airlines/star2000.csv.gz
2173762

real	0m42.464s
user	0m0.071s
sys	0m0.010s
```

With this PR
```
~ time mc select --query "select count(*) from S3Object" myminio/sjm-airlines/star2000.csv.gz
2173762

real	0m17.603s
user	0m0.093s
sys	0m0.008s
```

Almost a 250% improvement in performance. This PR avoids a lot of type
conversions and instead relies on raw sequences of data and interprets
them lazily.

```
benchcmp old new
benchmark                        old ns/op       new ns/op       delta
BenchmarkSQLAggregate_100K-4     551213          259782          -52.87%
BenchmarkSQLAggregate_1M-4       6981901985      2432413729      -65.16%
BenchmarkSQLAggregate_2M-4       13511978488     4536903552      -66.42%
BenchmarkSQLAggregate_10M-4      68427084908     23266283336     -66.00%

benchmark                        old allocs     new allocs     delta
BenchmarkSQLAggregate_100K-4     2366           485            -79.50%
BenchmarkSQLAggregate_1M-4       47455492       21462860       -54.77%
BenchmarkSQLAggregate_2M-4       95163637       43110771       -54.70%
BenchmarkSQLAggregate_10M-4      476959550      216906510      -54.52%

benchmark                        old bytes       new bytes      delta
BenchmarkSQLAggregate_100K-4     1233079         1086024        -11.93%
BenchmarkSQLAggregate_1M-4       2607984120      557038536      -78.64%
BenchmarkSQLAggregate_2M-4       5254103616      1128149168     -78.53%
BenchmarkSQLAggregate_10M-4      26443524872     5722715992     -78.36%
```

Makefile

@@ -23,8 +23,8 @@ verifiers: getdeps vet fmt lint cyclo deadcode spelling
 
 vet:
 	@echo "Running $@"
-	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult cmd
-	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult pkg
+	@go tool vet cmd
+	@go tool vet pkg
 
 fmt:
 	@echo "Running $@"

README.md

@@ -17,7 +17,7 @@ docker run -p 9000:9000 minio/minio server /data
 docker pull minio/minio:edge
 docker run -p 9000:9000 minio/minio:edge server /data
 ```
-Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
+Note: Docker will not display the autogenerated keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use autogenerated keys with containers. Please visit Minio Docker quickstart guide for more information [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
 ### Homebrew

cmd/acl-handlers.go

@@ -57,7 +57,7 @@ type accessControlPolicy struct {
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketACL")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -78,7 +78,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -92,7 +92,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -106,7 +106,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectACL")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -128,7 +128,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 	// Before proceeding validate if object exists.
 	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 

cmd/admin-handlers.go

@@ -53,9 +53,10 @@ type mgmtQueryKey string
 // Only valid query params for mgmt admin APIs.
 const (
 	mgmtBucket      mgmtQueryKey = "bucket"
-	mgmtPrefix      mgmtQueryKey = "prefix"
-	mgmtClientToken mgmtQueryKey = "clientToken"
-	mgmtForceStart  mgmtQueryKey = "forceStart"
+	mgmtPrefix                   = "prefix"
+	mgmtClientToken              = "clientToken"
+	mgmtForceStart               = "forceStart"
+	mgmtForceStop                = "forceStop"
 )
 
 var (
@@ -70,7 +71,9 @@ var (
 // -----------
 // Returns Administration API version
 func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	ctx := newContext(r, w, "Version")
+
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -78,8 +81,7 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request)
 
 	jsonBytes, err := json.Marshal(adminAPIVersionInfo)
 	if err != nil {
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		logger.LogIf(context.Background(), err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -90,7 +92,9 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request)
 // ----------
 // Returns server version and uptime.
 func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	ctx := newContext(r, w, "ServiceStatus")
+
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -106,8 +110,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 	// of read-quorum availability.
 	uptime, err := getPeerUptimes(globalAdminPeers)
 	if err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
-		logger.LogIf(context.Background(), err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -120,8 +123,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 	// Marshal API response
 	jsonBytes, err := json.Marshal(serverStatus)
 	if err != nil {
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		logger.LogIf(context.Background(), err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 	// Reply with storage information (across nodes in a
@@ -135,7 +137,9 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 // Restarts/Stops minio server gracefully. In a distributed setup,
 // restarts all the servers in the cluster.
 func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	ctx := newContext(r, w, "ServiceStopNRestart")
+
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -144,7 +148,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 	var sa madmin.ServiceAction
 	err := json.NewDecoder(r.Body).Decode(&sa)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
 		return
 	}
@@ -157,7 +161,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 		serviceSig = serviceStop
 	default:
 		writeErrorResponseJSON(w, ErrMalformedPOSTRequest, r.URL)
-		logger.LogIf(context.Background(), errors.New("Invalid service action received"))
+		logger.LogIf(ctx, errors.New("Invalid service action received"))
 		return
 	}
 
@@ -226,10 +230,12 @@ type ServerInfo struct {
 // ----------
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ServerInfo")
+
 	// Authenticate request
 
 	// Setting the region as empty so as the mc server info command is irrespective to the region.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -254,7 +260,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			serverInfoData, err := peer.cmdRunner.ServerInfo()
 			if err != nil {
 				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peer.addr)
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
+				ctx := logger.SetReqInfo(ctx, reqInfo)
 				logger.LogIf(ctx, err)
 				reply[idx].Error = err.Error()
 				return
@@ -269,8 +275,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	// Marshal API response
 	jsonBytes, err := json.Marshal(reply)
 	if err != nil {
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
-		logger.LogIf(context.Background(), err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -291,7 +296,9 @@ type StartProfilingResult struct {
 // ----------
 // Enable server profiling
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	ctx := newContext(r, w, "StartProfiling")
+
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -350,7 +357,9 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 // ----------
 // Download profiling information of all nodes in a zip format
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	ctx := newContext(r, w, "DownloadProfiling")
+
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -367,7 +376,7 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 		// Get profiling data from a node
 		data, err := peer.cmdRunner.DownloadProfilingData()
 		if err != nil {
-			logger.LogIf(context.Background(), fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
+			logger.LogIf(ctx, fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
 			continue
 		}
 
@@ -402,7 +411,7 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 
 // extractHealInitParams - Validates params for heal init API.
 func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
-	hs madmin.HealOpts, clientToken string, forceStart bool,
+	hs madmin.HealOpts, clientToken string, forceStart bool, forceStop bool,
 	err APIErrorCode) {
 
 	vars := mux.Vars(r)
@@ -433,7 +442,9 @@ func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 	if _, ok := qParms[string(mgmtForceStart)]; ok {
 		forceStart = true
 	}
-
+	if _, ok := qParms[string(mgmtForceStop)]; ok {
+		forceStop = true
+	}
 	// ignore body if clientToken is provided
 	if clientToken == "" {
 		jerr := json.NewDecoder(r.Body).Decode(&hs)
@@ -472,7 +483,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -484,7 +495,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bucket, objPrefix, hs, clientToken, forceStart, apiErr := extractHealInitParams(r)
+	bucket, objPrefix, hs, clientToken, forceStart, forceStop, apiErr := extractHealInitParams(r)
 	if apiErr != ErrNone {
 		writeErrorResponseJSON(w, apiErr, r.URL)
 		return
@@ -518,13 +529,35 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 				w.Write([]byte("\n\r"))
 				w.(http.Flusher).Flush()
 			case hr := <-respCh:
-				switch {
-				case hr.errCode == ErrNone:
-					writeSuccessResponseJSON(w, hr.respBytes)
-				case hr.errBody == "":
-					writeErrorResponseJSON(w, hr.errCode, r.URL)
+				switch hr.errCode {
+				case ErrNone:
+					if started {
+						w.Write(hr.respBytes)
+						w.(http.Flusher).Flush()
+					} else {
+						writeSuccessResponseJSON(w, hr.respBytes)
+					}
 				default:
-					writeCustomErrorResponseJSON(w, hr.errCode, hr.errBody, r.URL)
+					apiError := getAPIError(hr.errCode)
+					var errorRespJSON []byte
+					if hr.errBody == "" {
+						errorRespJSON = encodeResponseJSON(getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey)))
+					} else {
+						errorRespJSON = encodeResponseJSON(APIErrorResponse{
+							Code:      apiError.Code,
+							Message:   hr.errBody,
+							Resource:  r.URL.Path,
+							RequestID: w.Header().Get(responseRequestIDKey),
+							HostID:    "3L137",
+						})
+					}
+					if !started {
+						setCommonHeaders(w)
+						w.Header().Set("Content-Type", string(mimeJSON))
+						w.WriteHeader(apiError.HTTPStatusCode)
+					}
+					w.Write(errorRespJSON)
+					w.(http.Flusher).Flush()
 				}
 				break forLoop
 			}
@@ -535,34 +568,60 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	info := objLayer.StorageInfo(ctx)
 	numDisks := info.Backend.OfflineDisks + info.Backend.OnlineDisks
 
-	if clientToken == "" {
-		// Not a status request
-		nh := newHealSequence(bucket, objPrefix, handlers.GetSourceIP(r),
-			numDisks, hs, forceStart)
+	healPath := pathJoin(bucket, objPrefix)
+	if clientToken == "" && !forceStart && !forceStop {
+		nh, exists := globalAllHealState.getHealSequence(healPath)
+		if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
+			b, err := json.Marshal(madmin.HealStartSuccess{
+				ClientToken:   nh.clientToken,
+				ClientAddress: nh.clientAddress,
+				StartTime:     nh.startTime,
+			})
+			if err != nil {
+				writeErrorResponse(w, toAdminAPIErrCode(ctx, err), r.URL)
+				return
+			}
+			// Client token not specified but a heal sequence exists on a path,
+			// Send the token back to client.
+			writeSuccessResponseJSON(w, b)
+			return
+		}
+	}
 
-		respCh := make(chan healResp)
-		go func() {
-			respBytes, errCode, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
-			hr := healResp{respBytes, errCode, errMsg}
-			respCh <- hr
-		}()
-
-		// Due to the force-starting functionality, the Launch
-		// call above can take a long time - to keep the
-		// connection alive, we start sending whitespace
-		keepConnLive(w, respCh)
-	} else {
+	if clientToken != "" && !forceStart && !forceStop {
 		// Since clientToken is given, fetch heal status from running
 		// heal sequence.
-		path := bucket + "/" + objPrefix
 		respBytes, errCode := globalAllHealState.PopHealStatusJSON(
-			path, clientToken)
+			healPath, clientToken)
 		if errCode != ErrNone {
 			writeErrorResponseJSON(w, errCode, r.URL)
 		} else {
 			writeSuccessResponseJSON(w, respBytes)
 		}
+		return
 	}
+
+	respCh := make(chan healResp)
+	switch {
+	case forceStop:
+		go func() {
+			respBytes, errCode := globalAllHealState.stopHealSequence(healPath)
+			hr := healResp{respBytes: respBytes, errCode: errCode}
+			respCh <- hr
+		}()
+	case clientToken == "":
+		nh := newHealSequence(bucket, objPrefix, handlers.GetSourceIP(r), numDisks, hs, forceStart)
+		go func() {
+			respBytes, errCode, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
+			hr := healResp{respBytes, errCode, errMsg}
+			respCh <- hr
+		}()
+	}
+
+	// Due to the force-starting functionality, the Launch
+	// call above can take a long time - to keep the
+	// connection alive, we start sending whitespace
+	keepConnLive(w, respCh)
 }
 
 // GetConfigHandler - GET /minio/admin/v1/config
@@ -578,7 +637,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -586,22 +645,20 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	config, err := readServerConfig(ctx, objectAPI)
 	if err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
 	configData, err := json.MarshalIndent(config, "", "\t")
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
 	password := config.GetCredential().SecretKey
 	econfigData, err := madmin.EncryptData(password, configData)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -640,7 +697,7 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -655,14 +712,13 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 
 	config, err := readServerConfig(ctx, objectAPI)
 	if err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
 	configData, err := json.Marshal(config)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -684,8 +740,7 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	password := config.GetCredential().SecretKey
 	econfigData, err := madmin.EncryptData(password, []byte(newConfigStr))
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -694,12 +749,12 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 
 // toAdminAPIErrCode - converts errXLWriteQuorum error to admin API
 // specific error.
-func toAdminAPIErrCode(err error) APIErrorCode {
+func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
 	switch err {
 	case errXLWriteQuorum:
 		return ErrAdminConfigNoQuorum
 	default:
-		return toAPIErrorCode(err)
+		return toAPIErrorCode(ctx, err)
 	}
 }
 
@@ -715,7 +770,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -730,8 +785,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
 	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 	}
 }
 
@@ -747,7 +801,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -755,22 +809,20 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 
 	allCredentials, err := globalIAMSys.ListUsers()
 	if err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
 	data, err := json.Marshal(allCredentials)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
 	password := globalServerConfig.GetCredential().SecretKey
 	econfigData, err := madmin.EncryptData(password, data)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -789,7 +841,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -812,8 +864,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	}
 
 	if err := globalIAMSys.SetUserStatus(accessKey, madmin.AccountStatus(status)); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 }
@@ -830,7 +881,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -873,8 +924,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 }
@@ -891,7 +941,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -899,14 +949,12 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 
 	policies, err := globalIAMSys.ListCannedPolicies()
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
 	if err = json.NewEncoder(w).Encode(policies); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -928,7 +976,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	policyName := vars["name"]
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -941,8 +989,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	}
 
 	if err := globalIAMSys.DeleteCannedPolicy(policyName); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 }
@@ -962,7 +1009,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	policyName := vars["name"]
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -999,8 +1046,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	}
 
 	if err = globalIAMSys.SetCannedPolicy(policyName, *iamPolicy); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 }
@@ -1021,7 +1067,7 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 	policyName := vars["name"]
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -1040,8 +1086,7 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 	}
 
 	if err := globalIAMSys.SetUserPolicy(accessKey, policyName); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 	}
 }
 
@@ -1057,7 +1102,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -1120,7 +1165,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -1162,7 +1207,7 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -1171,15 +1216,14 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	// Load config
 	configStruct, err := readServerConfig(ctx, objectAPI)
 	if err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
 	// Convert config to json bytes
 	configBytes, err := json.Marshal(configStruct)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -1251,7 +1295,7 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -1282,7 +1326,7 @@ func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 	}
 
 	// Authenticate request
-	adminAPIErr := checkAdminRequestAuthType(r, "")
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -1312,7 +1356,7 @@ func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 
 	creds, err := auth.CreateCredentials(req.AccessKey, req.SecretKey)
 	if err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 
@@ -1327,7 +1371,7 @@ func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 	globalActiveCred = creds
 
 	if err = saveServerConfig(ctx, objectAPI, globalServerConfig); err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
 

cmd/admin-handlers_test.go

@@ -38,7 +38,7 @@ import (
 
 var (
 	configJSON = []byte(`{
-  "version": "31",
+  "version": "32",
   "credential": {
     "accessKey": "minio",
     "secretKey": "minio123"
@@ -157,6 +157,17 @@ var (
           "maxPubAcksInflight": 0
         }
       }
+	},
+    "nsq": {
+      "1": {
+        "enable": false,
+        "nsqdAddress": "",
+        "topic": "",
+        "tls": {
+			"enable": false,
+			"skipVerify": false
+		}
+      }
     },
     "postgresql": {
       "1": {
@@ -539,7 +550,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	credentials := globalServerConfig.GetCredential()
 
 	body, err := json.Marshal(madmin.ServiceAction{
-		cmd.toServiceActionValue()})
+		Action: cmd.toServiceActionValue()})
 	if err != nil {
 		t.Fatalf("JSONify error: %v", err)
 	}
@@ -746,7 +757,7 @@ func TestSetConfigHandler(t *testing.T) {
 	rec := httptest.NewRecorder()
 	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
-		t.Errorf("Expected to succeed but failed with %d", rec.Code)
+		t.Errorf("Expected to succeed but failed with %d, body: %s", rec.Code, rec.Body)
 	}
 
 	// Check that a very large config file returns an error.
@@ -856,12 +867,12 @@ func TestToAdminAPIErr(t *testing.T) {
 		// 3. Non-admin API specific error.
 		{
 			err:            errDiskNotFound,
-			expectedAPIErr: toAPIErrorCode(errDiskNotFound),
+			expectedAPIErr: toAPIErrorCode(context.Background(), errDiskNotFound),
 		},
 	}
 
 	for i, test := range testCases {
-		actualErr := toAdminAPIErrCode(test.err)
+		actualErr := toAdminAPIErrCode(context.Background(), test.err)
 		if actualErr != test.expectedAPIErr {
 			t.Errorf("Test %d: Expected %v but received %v",
 				i+1, test.expectedAPIErr, actualErr)

cmd/admin-heal-ops.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"runtime"
 	"strings"
 	"sync"
@@ -61,8 +62,8 @@ var (
 	errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
 	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
 
-	errFnHealFromAPIErr = func(err error) error {
-		errCode := toAPIErrorCode(err)
+	errFnHealFromAPIErr = func(ctx context.Context, err error) error {
+		errCode := toAPIErrorCode(ctx, err)
 		apiErr := getAPIError(errCode)
 		return fmt.Errorf("Heal internal error: %s: %s",
 			apiErr.Code, apiErr.Description)
@@ -123,6 +124,35 @@ func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists b
 	return h, exists
 }
 
+func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
+	var hsp madmin.HealStopSuccess
+	he, exists := ahs.getHealSequence(path)
+	if !exists {
+		hsp = madmin.HealStopSuccess{
+			ClientToken: "invalid",
+			StartTime:   UTCNow(),
+		}
+	} else {
+		hsp = madmin.HealStopSuccess{
+			ClientToken:   he.clientToken,
+			ClientAddress: he.clientAddress,
+			StartTime:     he.startTime,
+		}
+
+		he.stop()
+		for !he.hasEnded() {
+			time.Sleep(1 * time.Second)
+		}
+		ahs.Lock()
+		defer ahs.Unlock()
+		// Heal sequence explicitly stopped, remove it.
+		delete(ahs.healSeqMap, path)
+	}
+
+	b, err := json.Marshal(&hsp)
+	return b, toAdminAPIErrCode(context.Background(), err)
+}
+
 // LaunchNewHealSequence - launches a background routine that performs
 // healing according to the healSequence argument. For each heal
 // sequence, state is stored in the `globalAllHealState`, which is a
@@ -143,20 +173,20 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 			existsAndLive = true
 		}
 	}
+
 	if existsAndLive {
 		// A heal sequence exists on the given path.
 		if h.forceStarted {
-			// stop the running heal sequence - wait for
-			// it to finish.
+			// stop the running heal sequence - wait for it to finish.
 			he.stop()
 			for !he.hasEnded() {
-				time.Sleep(10 * time.Second)
+				time.Sleep(1 * time.Second)
 			}
 		} else {
 			errMsg = "Heal is already running on the given path " +
 				"(use force-start option to stop and start afresh). " +
-				fmt.Sprintf("The heal was started by IP %s at %s",
-					h.clientAddress, h.startTime)
+				fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
+					h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken)
 
 			return nil, ErrHealAlreadyRunning, errMsg
 		}
@@ -224,7 +254,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 		StartTime:     h.startTime,
 	})
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(h.ctx, err)
 		return nil, ErrInternalError, ""
 	}
 	return b, ErrNone, ""
@@ -272,7 +302,7 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 
 	jbytes, err := json.Marshal(h.currentStatus)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(h.ctx, err)
 		return nil, ErrInternalError
 	}
 
@@ -285,7 +315,7 @@ type healSequence struct {
 	// bucket, and prefix on which heal seq. was initiated
 	bucket, objPrefix string
 
-	// path is just bucket + "/" + objPrefix
+	// path is just pathJoin(bucket, objPrefix)
 	path string
 
 	// time at which heal sequence was started
@@ -330,7 +360,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 	return &healSequence{
 		bucket:        bucket,
 		objPrefix:     objPrefix,
-		path:          bucket + "/" + objPrefix,
+		path:          pathJoin(bucket, objPrefix),
 		startTime:     UTCNow(),
 		clientToken:   mustGetUUID(),
 		clientAddress: clientAddr,
@@ -552,7 +582,7 @@ func (h *healSequence) healConfig() error {
 			// before proceeding to heal
 			waitCount := 60
 			// Any requests in progress, delay the heal.
-			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
 				waitCount--
 				time.Sleep(1 * time.Second)
 			}
@@ -562,7 +592,7 @@ func (h *healSequence) healConfig() error {
 		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, minioConfigPrefix,
 			marker, "", 1000)
 		if err != nil {
-			return errFnHealFromAPIErr(err)
+			return errFnHealFromAPIErr(h.ctx, err)
 		}
 
 		for index := range objectInfos.Objects {
@@ -609,7 +639,7 @@ func (h *healSequence) healDiskFormat() error {
 	// return any error, ignore error returned when disks have
 	// already healed.
 	if err != nil && err != errNoHealRequired {
-		return errFnHealFromAPIErr(err)
+		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
 	// Healing succeeded notify the peers to reload format and re-initialize disks.
@@ -641,7 +671,7 @@ func (h *healSequence) healBuckets() error {
 
 	buckets, err := objectAPI.ListBucketsHeal(h.ctx)
 	if err != nil {
-		return errFnHealFromAPIErr(err)
+		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
 	for _, bucket := range buckets {
@@ -698,7 +728,7 @@ func (h *healSequence) healBucket(bucket string) error {
 			// before proceeding to heal
 			waitCount := 60
 			// Any requests in progress, delay the heal.
-			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
 				waitCount--
 				time.Sleep(1 * time.Second)
 			}
@@ -708,7 +738,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
 			h.objPrefix, marker, "", entries)
 		if err != nil {
-			return errFnHealFromAPIErr(err)
+			return errFnHealFromAPIErr(h.ctx, err)
 		}
 
 		g := errgroup.WithNErrs(len(objectInfos.Objects))

cmd/api-errors.go

@@ -23,11 +23,13 @@ import (
 	"net/http"
 
 	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/s3select"
+	"github.com/minio/minio/pkg/s3select/format"
 )
 
 // APIError structure
@@ -150,6 +152,9 @@ const (
 	ErrKMSNotConfigured
 	ErrKMSAuthFailure
 
+	ErrNoAccessKey
+	ErrInvalidToken
+
 	// Bucket notification related errors.
 	ErrEventNotification
 	ErrARNNotification
@@ -806,6 +811,16 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Server side encryption specified but KMS authorization failed",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrNoAccessKey: {
+		Code:           "AccessDenied",
+		Description:    "No AWSAccessKey was presented",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrInvalidToken: {
+		Code:           "InvalidTokenId",
+		Description:    "The security token included in the request is invalid",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 
 	/// S3 extensions.
 	ErrContentSHA256Mismatch: {
@@ -1427,7 +1442,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 // toAPIErrorCode - Converts embedded errors. Convenience
 // function written to handle all cases where we have known types of
 // errors returned by underlying layers.
-func toAPIErrorCode(err error) (apiErr APIErrorCode) {
+func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	if err == nil {
 		return ErrNone
 	}
@@ -1641,7 +1656,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEvaluatorBindingDoesNotExist
 	case s3select.ErrMissingHeaders:
 		apiErr = ErrMissingHeaders
-
+	case format.ErrParseInvalidPathComponent:
+		apiErr = ErrMissingHeaders
 	}
 
 	// Compression errors
@@ -1754,6 +1770,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectTampered
 	default:
 		apiErr = ErrInternalError
+		// Make sure to log the errors which we cannot translate
+		// to a meaningful S3 API errors. This is added to aid in
+		// debugging unexpected/unhandled errors.
+		logger.LogIf(ctx, err)
 	}
 
 	return apiErr

cmd/api-errors_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"testing"
 
@@ -64,8 +65,9 @@ var toAPIErrorCodeTests = []struct {
 }
 
 func TestAPIErrCode(t *testing.T) {
+	ctx := context.Background()
 	for i, testCase := range toAPIErrorCodeTests {
-		errCode := toAPIErrorCode(testCase.err)
+		errCode := toAPIErrorCode(ctx, testCase.err)
 		if errCode != testCase.errCode {
 			t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)
 		}

cmd/api-response-multipart.go

@@ -16,7 +16,10 @@
 
 package cmd
 
-import "net/http"
+import (
+	"context"
+	"net/http"
+)
 
 // Represents additional fields necessary for ErrPartTooSmall S3 error.
 type completeMultipartAPIError struct {
@@ -42,7 +45,7 @@ type completeMultipartAPIError struct {
 // of this function.
 func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
 
-	apiError := getAPIError(toAPIErrorCode(err))
+	apiError := getAPIError(toAPIErrorCode(context.Background(), err))
 	// Generate complete multipart error response.
 	errorResponse := getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey))
 	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}

cmd/api-response.go

@@ -608,7 +608,7 @@ func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode,
 		Code:      apiError.Code,
 		Message:   errBody,
 		Resource:  reqURL.Path,
-		RequestID: "3L137",
+		RequestID: w.Header().Get(responseRequestIDKey),
 		HostID:    "3L137",
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)

cmd/auth-handler.go

@@ -22,6 +22,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -29,6 +30,7 @@ import (
 
 	jwtgo "github.com/dgrijalva/jwt-go"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/policy"
@@ -107,17 +109,17 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypeJWT
 	} else if isRequestPostPolicySignatureV4(r) {
 		return authTypePostPolicy
-	} else if _, ok := r.Header["Authorization"]; !ok {
-		return authTypeAnonymous
 	} else if _, ok := r.URL.Query()["Action"]; ok {
 		return authTypeSTS
+	} else if _, ok := r.Header["Authorization"]; !ok {
+		return authTypeAnonymous
 	}
 	return authTypeUnknown
 }
 
 // checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
 // It does not accept presigned or JWT or anonymous requests.
-func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
+func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
 	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
 		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
@@ -134,11 +136,11 @@ func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 		}
 
 		// we only support V4 (no presign) with auth body
-		s3Err = isReqAuthenticated(r, region)
+		s3Err = isReqAuthenticated(ctx, r, region)
 	}
 	if s3Err != ErrNone {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		ctx := logger.SetReqInfo(ctx, reqInfo)
 		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description))
 	}
 	return s3Err
@@ -154,19 +156,48 @@ func getSessionToken(r *http.Request) (token string) {
 }
 
 // Fetch claims in the security token returned by the client and validate the token.
-func getClaimsFromToken(r *http.Request) (map[string]interface{}, APIErrorCode) {
+func getClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
+	stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
+		if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
+			return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
+		}
+		if err := jwtToken.Claims.Valid(); err != nil {
+			return nil, errAuthentication
+		}
+		if claims, ok := jwtToken.Claims.(jwtgo.MapClaims); ok {
+			if _, ok = claims["accessKey"].(string); !ok {
+				return nil, errInvalidAccessKeyID
+			}
+			// JWT token for x-amz-security-token is signed with admin
+			// secret key, temporary credentials become invalid if
+			// server admin credentials change. This is done to ensure
+			// that clients cannot decode the token using the temp
+			// secret keys and generate an entirely new claim by essentially
+			// hijacking the policies. We need to make sure that this is
+			// based an admin credential such that token cannot be decoded
+			// on the client side and is treated like an opaque value.
+			return []byte(globalServerConfig.GetCredential().SecretKey), nil
+		}
+		return nil, errAuthentication
+	}
 	claims := make(map[string]interface{})
 	token := getSessionToken(r)
 	if token == "" {
 		return nil, ErrNone
 	}
+	if token != "" && cred.AccessKey == "" {
+		return nil, ErrNoAccessKey
+	}
+	if token != cred.SessionToken {
+		return nil, ErrInvalidToken
+	}
 	p := &jwtgo.Parser{}
 	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
 	if err != nil {
-		return nil, toAPIErrorCode(errAuthentication)
+		return nil, toAPIErrorCode(context.Background(), errAuthentication)
 	}
 	if !jtoken.Valid {
-		return nil, toAPIErrorCode(errAuthentication)
+		return nil, toAPIErrorCode(context.Background(), errAuthentication)
 	}
 	return claims, ErrNone
 }
@@ -177,7 +208,7 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, APIErrorCode)
 //   for authenticated requests validates IAM policies.
 // returns APIErrorCode if any to be replied to the client.
 func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
-	var accessKey string
+	var cred auth.Credentials
 	var owner bool
 	switch getRequestAuthType(r) {
 	case authTypeUnknown, authTypeStreamingSigned:
@@ -186,17 +217,17 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
 			return s3Err
 		}
-		accessKey, owner, s3Err = getReqAccessKeyV2(r)
+		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
 		region := globalServerConfig.GetRegion()
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
 		}
-		if s3Err = isReqAuthenticated(r, region); s3Err != ErrNone {
+		if s3Err = isReqAuthenticated(ctx, r, region); s3Err != ErrNone {
 			return s3Err
 		}
-		accessKey, owner, s3Err = getReqAccessKeyV4(r, region)
+		cred, owner, s3Err = getReqAccessKeyV4(r, region)
 	}
 	if s3Err != ErrNone {
 		return s3Err
@@ -225,14 +256,14 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
 
-	claims, s3Err := getClaimsFromToken(r)
+	claims, s3Err := getClaimsFromToken(r, cred)
 	if s3Err != ErrNone {
 		return s3Err
 	}
 
-	if accessKey == "" {
+	if cred.AccessKey == "" {
 		if globalPolicySys.IsAllowed(policy.Args{
-			AccountName:     accessKey,
+			AccountName:     cred.AccessKey,
 			Action:          action,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, locationConstraint),
@@ -245,7 +276,7 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 	}
 
 	if globalIAMSys.IsAllowed(iampolicy.Args{
-		AccountName:     accessKey,
+		AccountName:     cred.AccessKey,
 		Action:          iampolicy.Action(action),
 		BucketName:      bucketName,
 		ConditionValues: getConditionValues(r, ""),
@@ -279,7 +310,7 @@ func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode)
 }
 
 // Verify if request has valid AWS Signature Version '4'.
-func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
+func isReqAuthenticated(ctx context.Context, r *http.Request, region string) (s3Error APIErrorCode) {
 	if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
 		return errCode
 	}
@@ -316,7 +347,7 @@ func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
 	// The verification happens implicit during reading.
 	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1)
 	if err != nil {
-		return toAPIErrorCode(err)
+		return toAPIErrorCode(ctx, err)
 	}
 	r.Body = ioutil.NopCloser(reader)
 	return ErrNone
@@ -364,6 +395,9 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 		a.handler.ServeHTTP(w, r)
 		return
+	} else if aType == authTypeSTS {
+		a.handler.ServeHTTP(w, r)
+		return
 	}
 	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
 }
@@ -372,29 +406,29 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.
 func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request) (s3Err APIErrorCode) {
-	var accessKey string
+	var cred auth.Credentials
 	var owner bool
 	switch atype {
 	case authTypeUnknown:
 		return ErrAccessDenied
 	case authTypeSignedV2, authTypePresignedV2:
-		accessKey, owner, s3Err = getReqAccessKeyV2(r)
+		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
 		region := globalServerConfig.GetRegion()
-		accessKey, owner, s3Err = getReqAccessKeyV4(r, region)
+		cred, owner, s3Err = getReqAccessKeyV4(r, region)
 	}
 	if s3Err != ErrNone {
 		return s3Err
 	}
 
-	claims, s3Err := getClaimsFromToken(r)
+	claims, s3Err := getClaimsFromToken(r, cred)
 	if s3Err != ErrNone {
 		return s3Err
 	}
 
-	if accessKey == "" {
+	if cred.AccessKey == "" {
 		if globalPolicySys.IsAllowed(policy.Args{
-			AccountName:     accessKey,
+			AccountName:     cred.AccessKey,
 			Action:          policy.PutObjectAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, ""),
@@ -407,7 +441,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
 	}
 
 	if globalIAMSys.IsAllowed(iampolicy.Args{
-		AccountName:     accessKey,
+		AccountName:     cred.AccessKey,
 		Action:          policy.PutObjectAction,
 		BucketName:      bucketName,
 		ConditionValues: getConditionValues(r, ""),

cmd/auth-handler_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -377,11 +378,12 @@ func TestIsReqAuthenticated(t *testing.T) {
 		{mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrNone},
 	}
 
+	ctx := context.Background()
 	// Validates all testcases.
 	for i, testCase := range testCases {
-		if s3Error := isReqAuthenticated(testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
-			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(err) != testCase.s3Error {
-				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(err))
+		if s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
+			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
+				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(ctx, err))
 			}
 		}
 	}
@@ -413,8 +415,9 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		{Request: mustNewPresignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 		{Request: mustNewPresignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 	}
+	ctx := context.Background()
 	for i, testCase := range testCases {
-		if s3Error := checkAdminRequestAuthType(testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
+		if s3Error := checkAdminRequestAuthType(ctx, testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}

cmd/bucket-handlers-listobjects.go

@@ -59,7 +59,7 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -100,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	// marshaled into S3 compatible XML header.
 	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -118,7 +118,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
 			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}
@@ -140,7 +140,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -182,7 +182,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -200,7 +200,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
 			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}

cmd/bucket-handlers.go

@@ -89,7 +89,7 @@ func initFederatorBackend(objLayer ObjectLayer) {
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLocation")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -110,7 +110,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -139,7 +139,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -174,7 +174,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 
 	listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	// generate response
@@ -192,7 +192,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBuckets")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -214,7 +214,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	if globalDNSConfig != nil {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && err != dns.ErrNoEntriesFound {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 		bucketSet := set.NewStringSet()
@@ -233,7 +233,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		var err error
 		bucketsInfo, err = listBuckets(ctx)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 	}
@@ -250,6 +250,8 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteMultipleObjects")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -352,8 +354,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 		// Error during delete should be collected separately.
 		deleteErrors = append(deleteErrors, DeleteError{
-			Code:    errorCodeResponse[toAPIErrorCode(err)].Code,
-			Message: errorCodeResponse[toAPIErrorCode(err)].Description,
+			Code:    errorCodeResponse[toAPIErrorCode(ctx, err)].Code,
+			Message: errorCodeResponse[toAPIErrorCode(ctx, err)].Description,
 			Key:     object.ObjectName,
 		})
 	}
@@ -380,10 +382,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			Object: ObjectInfo{
 				Name: dobj.ObjectName,
 			},
-			ReqParams: extractReqParams(r),
-			UserAgent: r.UserAgent(),
-			Host:      host,
-			Port:      port,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         host,
+			Port:         port,
 		})
 	}
 }
@@ -394,7 +397,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucket")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -429,12 +432,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
-					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 					return
 				}
 				if err = globalDNSConfig.Put(bucket); err != nil {
 					objectAPI.DeleteBucket(ctx, bucket)
-					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 					return
 				}
 
@@ -444,7 +447,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				writeSuccessResponseHeadersOnly(w)
 				return
 			}
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 
 		}
@@ -455,7 +458,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -472,7 +475,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PostPolicyBucket")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -586,12 +589,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	lengthRange := postPolicyForm.Conditions.ContentLengthRange
 	if lengthRange.Valid {
 		if fileSize < lengthRange.Min {
-			writeErrorResponse(w, toAPIErrorCode(errDataTooSmall), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL)
 			return
 		}
 
 		if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
-			writeErrorResponse(w, toAPIErrorCode(errDataTooLarge), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL)
 			return
 		}
 	}
@@ -607,7 +610,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -618,19 +621,19 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			if crypto.SSEC.IsRequested(formValues) {
 				key, err = ParseSSECustomerHeader(formValues)
 				if err != nil {
-					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 					return
 				}
 			}
 			reader, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 			info := ObjectInfo{Size: fileSize}
 			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}
@@ -638,7 +641,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -654,13 +657,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedPost,
-		BucketName: objInfo.Bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedPost,
+		BucketName:   objInfo.Bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 
 	if successRedirect != "" {
@@ -685,12 +689,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	default:
 		writeSuccessNoContent(w)
 	}
-
-	for k, v := range objInfo.UserDefined {
-		logger.GetReqInfo(ctx).SetTags(k, v)
-	}
-
-	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // HeadBucketHandler - HEAD Bucket
@@ -702,7 +700,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadBucket")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -723,7 +721,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
+		writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
 		return
 	}
 
@@ -734,7 +732,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucket")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -756,7 +754,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 	// Attempt to delete bucket.
 	if err := deleteBucket(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -768,7 +766,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		if err := globalDNSConfig.Delete(bucket); err != nil {
 			// Deleting DNS entry failed, attempt to create the bucket again.
 			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 	}

cmd/bucket-notification-handlers.go

@@ -44,7 +44,7 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketNotification")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
@@ -67,7 +67,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	_, err := objAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -76,17 +76,21 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 	if err != nil {
 		// Ignore errNoSuchNotifications to comply with AWS S3.
 		if err != errNoSuchNotifications {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 
 		nConfig = &event.Config{}
 	}
 
+	// If xml namespace is empty, set a default value before returning.
+	if nConfig.XMLNS == "" {
+		nConfig.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
+	}
+
 	notificationBytes, err := xml.Marshal(nConfig)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -98,7 +102,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketNotification")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -121,7 +125,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -136,7 +140,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	if err != nil {
 		apiErr := ErrMalformedXML
 		if event.IsEventError(err) {
-			apiErr = toAPIErrorCode(err)
+			apiErr = toAPIErrorCode(ctx, err)
 		}
 
 		writeErrorResponse(w, apiErr, r.URL)
@@ -144,7 +148,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	}
 
 	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -160,7 +164,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListenBucketNotification")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
@@ -189,7 +193,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 	if len(values["prefix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 
@@ -202,7 +206,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 	if len(values["suffix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 
@@ -215,7 +219,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	for _, s := range values["events"] {
 		eventName, err := event.ParseName(s)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 
@@ -223,19 +227,19 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 
 	if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
 	host, err := xnet.ParseHost(r.RemoteAddr)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
 	target, err := target.NewHTTPClientTarget(*host, w)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -243,8 +247,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
@@ -252,14 +255,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
 	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -269,8 +271,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 }

cmd/bucket-policy-handlers.go

@@ -40,7 +40,7 @@ const (
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketPolicy")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -58,7 +58,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -88,7 +88,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -103,7 +103,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketPolicy")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -121,12 +121,12 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -141,7 +141,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketPolicy")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -159,21 +159,20 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
 	// Read bucket access policy.
 	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
 	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 

cmd/certs.go

@@ -150,24 +150,20 @@ func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	return cert, nil
 }
 
-func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, c *certs.Certs, secureConn bool, err error) {
+func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return nil, nil, nil, false, nil
+		return nil, nil, false, nil
 	}
 
 	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return nil, nil, nil, false, err
+		return nil, nil, false, err
 	}
 
 	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
 	if err != nil {
-		return nil, nil, nil, false, err
-	}
-
-	if rootCAs, err = getRootCAs(getCADir()); err != nil {
-		return nil, nil, nil, false, err
+		return nil, nil, false, err
 	}
 
 	secureConn = true
-	return x509Certs, rootCAs, c, secureConn, nil
+	return x509Certs, c, secureConn, nil
 }

cmd/common-main.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"errors"
 	"net"
 	"os"
@@ -33,6 +34,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Check for updates and print a notification message
@@ -49,10 +51,23 @@ func checkUpdate(mode string) {
 
 // Load logger targets based on user's configuration
 func loadLoggers() {
-	if endpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT"); ok {
-		// Enable http logging through ENV, this is specifically added gateway audit logging.
-		logger.AddTarget(logger.NewHTTP(endpoint, NewCustomHTTPTransport()))
-		return
+	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable audit HTTP logging through ENV.
+		logger.AddAuditTarget(logger.NewHTTP(auditEndpoint, NewCustomHTTPTransport()))
+	}
+
+	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable HTTP logging through ENV.
+		logger.AddTarget(logger.NewHTTP(loggerEndpoint, NewCustomHTTPTransport()))
+	} else {
+		for _, l := range globalServerConfig.Logger.HTTP {
+			if l.Enabled {
+				// Enable http logging
+				logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
+			}
+		}
 	}
 
 	if globalServerConfig.Logger.Console.Enabled {
@@ -60,12 +75,6 @@ func loadLoggers() {
 		logger.AddTarget(logger.NewConsole())
 	}
 
-	for _, l := range globalServerConfig.Logger.HTTP {
-		if l.Enabled {
-			// Enable http logging
-			logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
-		}
-	}
 }
 
 func handleCommonCmdArgs(ctx *cli.Context) {
@@ -157,12 +166,46 @@ func handleCommonEnvVars() {
 	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
 	if ok {
 		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+
+		var etcdSecure bool
+		for _, endpoint := range etcdEndpoints {
+			u, err := xnet.ParseURL(endpoint)
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+			}
+			// If one of the endpoint is https, we will use https directly.
+			etcdSecure = etcdSecure || u.Scheme == "https"
+		}
+
 		var err error
-		globalEtcdClient, err = etcd.New(etcd.Config{
-			Endpoints:         etcdEndpoints,
-			DialTimeout:       defaultDialTimeout,
-			DialKeepAliveTime: defaultDialKeepAlive,
-		})
+		if etcdSecure {
+			// This is only to support client side certificate authentication
+			// https://coreos.com/etcd/docs/latest/op-guide/security.html
+			etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT")
+			etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY")
+			var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+			if ok1 && ok2 {
+				getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+					cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
+					return &cert, terr
+				}
+			}
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+				TLS: &tls.Config{
+					RootCAs:              globalRootCAs,
+					GetClientCertificate: getClientCertificate,
+				},
+			})
+		} else {
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+			})
+		}
 		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
 	}
 

cmd/config-current.go

@@ -43,9 +43,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "31"
+const serverConfigVersion = "32"
 
-type serverConfig = serverConfigV31
+type serverConfig = serverConfigV32
 
 var (
 	// globalServerConfig server config.
@@ -210,6 +210,12 @@ func (s *serverConfig) Validate() error {
 		}
 	}
 
+	for _, v := range s.Notify.NSQ {
+		if err := v.Validate(); err != nil {
+			return fmt.Errorf("nsq: %s", err)
+		}
+	}
+
 	for _, v := range s.Notify.PostgreSQL {
 		if err := v.Validate(); err != nil {
 			return fmt.Errorf("postgreSQL: %s", err)
@@ -358,6 +364,17 @@ func (s *serverConfig) TestNotificationTargets() error {
 		t.Close()
 	}
 
+	for k, v := range s.Notify.NSQ {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewNSQTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("nsq(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
 	for k, v := range s.Notify.PostgreSQL {
 		if !v.Enable {
 			continue
@@ -405,6 +422,8 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "AMQP Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
 		return "NATS Notification configuration differs"
+	case !reflect.DeepEqual(s.Notify.NSQ, t.Notify.NSQ):
+		return "NSQ Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.Elasticsearch, t.Notify.Elasticsearch):
 		return "ElasticSearch Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.Redis, t.Notify.Redis):
@@ -470,6 +489,8 @@ func newServerConfig() *serverConfig {
 	srvCfg.Notify.Redis["1"] = target.RedisArgs{}
 	srvCfg.Notify.NATS = make(map[string]target.NATSArgs)
 	srvCfg.Notify.NATS["1"] = target.NATSArgs{}
+	srvCfg.Notify.NSQ = make(map[string]target.NSQArgs)
+	srvCfg.Notify.NSQ["1"] = target.NSQArgs{}
 	srvCfg.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
 	srvCfg.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{}
 	srvCfg.Notify.MySQL = make(map[string]target.MySQLArgs)
@@ -705,6 +726,20 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 		}
 	}
 
+	for id, args := range config.Notify.NSQ {
+		if args.Enable {
+			newTarget, err := target.NewNSQTarget(id, args)
+			if err != nil {
+				logger.LogIf(context.Background(), err)
+				continue
+			}
+			if err = targetList.Add(newTarget); err != nil {
+				logger.LogIf(context.Background(), err)
+				continue
+			}
+		}
+	}
+
 	for id, args := range config.Notify.PostgreSQL {
 		if args.Enable {
 			newTarget, err := target.NewPostgreSQLTarget(id, args)

cmd/config-current_test.go

@@ -234,6 +234,9 @@ func TestValidateConfig(t *testing.T) {
 
 		// Test 27 - Test MQTT
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
+
+		// Test 28 - Test NSQ
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},
 	}
 
 	for i, testCase := range testCases {
@@ -293,48 +296,54 @@ func TestConfigDiff(t *testing.T) {
 			"NATS Notification configuration differs",
 		},
 		// 7
+		{
+			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: true}}}},
+			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: false}}}},
+			"NSQ Notification configuration differs",
+		},
+		// 8
 		{
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
 			"ElasticSearch Notification configuration differs",
 		},
-		// 8
+		// 9
 		{
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
 			"Redis Notification configuration differs",
 		},
-		// 9
+		// 10
 		{
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
 			"PostgreSQL Notification configuration differs",
 		},
-		// 10
+		// 11
 		{
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
 			"Kafka Notification configuration differs",
 		},
-		// 11
+		// 12
 		{
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
 			"Webhook Notification configuration differs",
 		},
-		// 12
+		// 13
 		{
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
 			"MySQL Notification configuration differs",
 		},
-		// 13
+		// 14
 		{
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
 			"MQTT Notification configuration differs",
 		},
-		// 14
+		// 15
 		{
 			&serverConfig{Logger: loggerConfig{
 				Console: loggerConsole{Enabled: true},

cmd/config-migrate.go

@@ -919,7 +919,7 @@ func migrateV12ToV13() error {
 	// Copy over fields from V12 into V13 config struct
 	srvConfig := &serverConfigV13{
 		Logger: &loggerV7{},
-		Notify: &notifier{},
+		Notify: &notifierV3{},
 	}
 	srvConfig.Version = "13"
 	srvConfig.Credential = cv12.Credential
@@ -999,7 +999,7 @@ func migrateV13ToV14() error {
 	// Copy over fields from V13 into V14 config struct
 	srvConfig := &serverConfigV14{
 		Logger: &loggerV7{},
-		Notify: &notifier{},
+		Notify: &notifierV3{},
 	}
 	srvConfig.Version = "14"
 	srvConfig.Credential = cv13.Credential
@@ -1084,7 +1084,7 @@ func migrateV14ToV15() error {
 	// Copy over fields from V14 into V15 config struct
 	srvConfig := &serverConfigV15{
 		Logger: &loggerV7{},
-		Notify: &notifier{},
+		Notify: &notifierV3{},
 	}
 	srvConfig.Version = "15"
 	srvConfig.Credential = cv14.Credential
@@ -1174,7 +1174,7 @@ func migrateV15ToV16() error {
 	// Copy over fields from V15 into V16 config struct
 	srvConfig := &serverConfigV16{
 		Logger: &loggers{},
-		Notify: &notifier{},
+		Notify: &notifierV3{},
 	}
 	srvConfig.Version = "16"
 	srvConfig.Credential = cv15.Credential
@@ -1264,7 +1264,7 @@ func migrateV16ToV17() error {
 	// Copy over fields from V16 into V17 config struct
 	srvConfig := &serverConfigV17{
 		Logger: &loggers{},
-		Notify: &notifier{},
+		Notify: &notifierV3{},
 	}
 	srvConfig.Version = "17"
 	srvConfig.Credential = cv16.Credential
@@ -1385,7 +1385,7 @@ func migrateV17ToV18() error {
 	// Copy over fields from V17 into V18 config struct
 	srvConfig := &serverConfigV17{
 		Logger: &loggers{},
-		Notify: &notifier{},
+		Notify: &notifierV3{},
 	}
 	srvConfig.Version = "18"
 	srvConfig.Credential = cv17.Credential
@@ -1487,7 +1487,7 @@ func migrateV18ToV19() error {
 	// Copy over fields from V18 into V19 config struct
 	srvConfig := &serverConfigV18{
 		Logger: &loggers{},
-		Notify: &notifier{},
+		Notify: &notifierV3{},
 	}
 	srvConfig.Version = "19"
 	srvConfig.Credential = cv18.Credential
@@ -1593,7 +1593,7 @@ func migrateV19ToV20() error {
 	// Copy over fields from V19 into V20 config struct
 	srvConfig := &serverConfigV20{
 		Logger: &loggers{},
-		Notify: &notifier{},
+		Notify: &notifierV3{},
 	}
 	srvConfig.Version = "20"
 	srvConfig.Credential = cv19.Credential
@@ -1697,7 +1697,7 @@ func migrateV20ToV21() error {
 
 	// Copy over fields from V20 into V21 config struct
 	srvConfig := &serverConfigV21{
-		Notify: &notifier{},
+		Notify: &notifierV3{},
 	}
 	srvConfig.Version = "21"
 	srvConfig.Credential = cv20.Credential
@@ -1801,7 +1801,7 @@ func migrateV21ToV22() error {
 
 	// Copy over fields from V21 into V22 config struct
 	srvConfig := &serverConfigV22{
-		Notify: notifier{},
+		Notify: notifierV3{},
 	}
 	srvConfig.Version = "22"
 	srvConfig.Credential = cv21.Credential
@@ -1905,7 +1905,7 @@ func migrateV22ToV23() error {
 
 	// Copy over fields from V22 into V23 config struct
 	srvConfig := &serverConfigV23{
-		Notify: notifier{},
+		Notify: notifierV3{},
 	}
 	srvConfig.Version = "23"
 	srvConfig.Credential = cv22.Credential
@@ -2018,7 +2018,7 @@ func migrateV23ToV24() error {
 
 	// Copy over fields from V23 into V24 config struct
 	srvConfig := &serverConfigV24{
-		Notify: notifier{},
+		Notify: notifierV3{},
 	}
 	srvConfig.Version = "24"
 	srvConfig.Credential = cv23.Credential
@@ -2131,7 +2131,7 @@ func migrateV24ToV25() error {
 
 	// Copy over fields from V24 into V25 config struct
 	srvConfig := &serverConfigV25{
-		Notify: notifier{},
+		Notify: notifierV3{},
 	}
 	srvConfig.Version = "25"
 	srvConfig.Credential = cv24.Credential
@@ -2249,7 +2249,7 @@ func migrateV25ToV26() error {
 
 	// Copy over fields from V25 into V26 config struct
 	srvConfig := &serverConfigV26{
-		Notify: notifier{},
+		Notify: notifierV3{},
 	}
 	srvConfig.Version = "26"
 	srvConfig.Credential = cv25.Credential
@@ -2413,7 +2413,7 @@ func migrateV27ToV28() error {
 	return nil
 }
 
-// Migrates '.minio.sys/config.json' to v31.
+// Migrates '.minio.sys/config.json' to v32.
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
 		return err
@@ -2424,7 +2424,10 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	if err := migrateV29ToV30MinioSys(objAPI); err != nil {
 		return err
 	}
-	return migrateV30ToV31MinioSys(objAPI)
+	if err := migrateV30ToV31MinioSys(objAPI); err != nil {
+		return err
+	}
+	return migrateV31ToV32MinioSys(objAPI)
 }
 
 func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
@@ -2585,3 +2588,38 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
 	logger.Info(configMigrateMSGTemplate, configFile, "30", "31")
 	return nil
 }
+
+func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "31")
+	if err == errConfigNotFound {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+	if !ok {
+		return nil
+	}
+
+	cfg := &serverConfigV32{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	cfg.Version = "32"
+	cfg.Notify.NSQ = make(map[string]target.NSQArgs)
+	cfg.Notify.NSQ["1"] = target.NSQArgs{}
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "31", "32")
+	return nil
+}

cmd/config-versions.go

@@ -373,7 +373,7 @@ type serverConfigV12 struct {
 	Notify notifierV2 `json:"notify"`
 }
 
-type notifier struct {
+type notifierV3 struct {
 	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
 	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
 	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
@@ -398,7 +398,7 @@ type serverConfigV13 struct {
 	Logger *loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV14 server configuration version '14' which is like
@@ -415,7 +415,7 @@ type serverConfigV14 struct {
 	Logger *loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV15 server configuration version '15' which is like
@@ -432,7 +432,7 @@ type serverConfigV15 struct {
 	Logger *loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // FileLogger is introduced to workaround the dependency about logrus
@@ -470,7 +470,7 @@ type serverConfigV16 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV17 server configuration version '17' which is like
@@ -489,7 +489,7 @@ type serverConfigV17 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV18 server configuration version '18' which is like
@@ -508,7 +508,7 @@ type serverConfigV18 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV19 server configuration version '19' which is like
@@ -526,7 +526,7 @@ type serverConfigV19 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV20 server configuration version '20' which is like
@@ -545,7 +545,7 @@ type serverConfigV20 struct {
 	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV21 is just like version '20' without logger field
@@ -560,7 +560,7 @@ type serverConfigV21 struct {
 	Domain     string           `json:"domain"`
 
 	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
+	Notify *notifierV3 `json:"notify"`
 }
 
 // serverConfigV22 is just like version '21' with added support
@@ -581,7 +581,7 @@ type serverConfigV22 struct {
 	StorageClass storageClassConfig `json:"storageclass"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV23 is just like version '22' with addition of cache field.
@@ -604,7 +604,7 @@ type serverConfigV23 struct {
 	Cache CacheConfig `json:"cache"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV24 is just like version '23', we had to revert
@@ -628,7 +628,7 @@ type serverConfigV24 struct {
 	Cache CacheConfig `json:"cache"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV25 is just like version '24', stores additionally
@@ -655,7 +655,7 @@ type serverConfigV25 struct {
 	Cache CacheConfig `json:"cache"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV26 is just like version '25', stores additionally
@@ -679,7 +679,7 @@ type serverConfigV26 struct {
 	Cache CacheConfig `json:"cache"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 }
 
 type loggerConsole struct {
@@ -720,7 +720,7 @@ type serverConfigV27 struct {
 	Cache CacheConfig `json:"cache"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
 	Logger loggerConfig `json:"logger"`
@@ -751,7 +751,7 @@ type serverConfigV28 struct {
 	KMS crypto.KMSConfig `json:"kms"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
 	Logger loggerConfig `json:"logger"`
@@ -787,7 +787,7 @@ type serverConfigV30 struct {
 	KMS crypto.KMSConfig `json:"kms"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
 	Logger loggerConfig `json:"logger"`
@@ -814,6 +814,61 @@ type serverConfigV31 struct {
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
 
+	// Notification queue configuration.
+	Notify notifierV3 `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+
+	// Compression configuration
+	Compression compressionConfig `json:"compress"`
+
+	// OpenID configuration
+	OpenID struct {
+		// JWKS validator config.
+		JWKS validator.JWKSArgs `json:"jwks"`
+	} `json:"openid"`
+
+	// External policy enforcements.
+	Policy struct {
+		// OPA configuration.
+		OPA iampolicy.OpaArgs `json:"opa"`
+
+		// Add new external policy enforcements here.
+	} `json:"policy"`
+}
+
+type notifier struct {
+	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
+	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
+	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
+	MQTT          map[string]target.MQTTArgs          `json:"mqtt"`
+	MySQL         map[string]target.MySQLArgs         `json:"mysql"`
+	NATS          map[string]target.NATSArgs          `json:"nats"`
+	NSQ           map[string]target.NSQArgs           `json:"nsq"`
+	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
+	Redis         map[string]target.RedisArgs         `json:"redis"`
+	Webhook       map[string]target.WebhookArgs       `json:"webhook"`
+}
+
+// serverConfigV32 is just like version '31' with added nsq notifer.
+type serverConfigV32 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
 	// Notification queue configuration.
 	Notify notifier `json:"notify"`
 

cmd/encryption-v1.go

@@ -156,6 +156,30 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 		sealedKey = objectKey.Seal(extKey, sealedKey.IV, crypto.SSEC.String(), bucket, object)
 		crypto.SSEC.CreateMetadata(metadata, sealedKey)
 		return nil
+	case crypto.S3.IsEncrypted(metadata):
+		if globalKMS == nil {
+			return errKMSNotConfigured
+		}
+		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
+		if err != nil {
+			return err
+		}
+		oldKey, err := globalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
+		if err != nil {
+			return err
+		}
+		var objectKey crypto.ObjectKey
+		if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
+			return err
+		}
+
+		newKey, encKey, err := globalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+		if err != nil {
+			return err
+		}
+		sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
+		crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
+		return nil
 	}
 }
 
@@ -332,7 +356,7 @@ func DecryptRequestWithSequenceNumberR(client io.Reader, h http.Header, bucket,
 
 // DecryptCopyRequestR - same as DecryptCopyRequest, but with a
 // Reader
-func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, metadata map[string]string) (io.Reader, error) {
+func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
 	var (
 		key []byte
 		err error
@@ -343,7 +367,7 @@ func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string,
 			return nil, err
 		}
 	}
-	return newDecryptReader(client, key, bucket, object, 0, metadata)
+	return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
 }
 
 func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
@@ -365,17 +389,6 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
 	return reader, nil
 }
 
-// GetEncryptedOffsetLength - returns encrypted offset and length
-// along with sequence number
-func GetEncryptedOffsetLength(startOffset, length int64, objInfo ObjectInfo) (seqNumber uint32, encStartOffset, encLength int64) {
-	if !isEncryptedMultipart(objInfo) {
-		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
-		return
-	}
-	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
-	return
-}
-
 // DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
 // reader
 func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
@@ -389,7 +402,7 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
 		var reader io.Reader
 		var err error
 		if copySource {
-			reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, oi.UserDefined)
+			reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
 		} else {
 			reader, err = DecryptRequestWithSequenceNumberR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
 		}
@@ -917,26 +930,29 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
 	}
 
 	// Assemble slice of (decrypted) part sizes in `sizes`
+	var sizes []int64
 	var decObjSize int64 // decrypted total object size
-	var partSize uint64
-	partSize, err = sio.DecryptedSize(uint64(o.Size))
-	if err != nil {
-		return
-	}
-	sizes := []int64{int64(partSize)}
-	decObjSize = sizes[0]
 	if isEncryptedMultipart(*o) {
 		sizes = make([]int64, len(o.Parts))
-		decObjSize = 0
 		for i, part := range o.Parts {
+			var partSize uint64
 			partSize, err = sio.DecryptedSize(uint64(part.Size))
 			if err != nil {
+				err = errObjectTampered
 				return
 			}
-			t := int64(partSize)
-			sizes[i] = t
-			decObjSize += t
+			sizes[i] = int64(partSize)
+			decObjSize += int64(partSize)
 		}
+	} else {
+		var partSize uint64
+		partSize, err = sio.DecryptedSize(uint64(o.Size))
+		if err != nil {
+			err = errObjectTampered
+			return
+		}
+		sizes = []int64{int64(partSize)}
+		decObjSize = sizes[0]
 	}
 
 	var off, length int64
@@ -1046,7 +1062,7 @@ func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErr
 		}
 		var err error
 		if info.Size, err = info.DecryptedSize(); err != nil {
-			apiErr = toAPIErrorCode(err)
+			apiErr = toAPIErrorCode(context.Background(), err)
 		}
 	}
 	return

cmd/encryption-v1_test.go

@@ -37,7 +37,7 @@ var hasServerSideEncryptionHeaderTests = []struct {
 	{headers: map[string]string{}, sseRequest: false},                                                                                                           // 4
 	{headers: map[string]string{crypto.SSECopyAlgorithm + " ": "AES256", " " + crypto.SSECopyKey: "key", crypto.SSECopyKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
 	{headers: map[string]string{crypto.SSECopyAlgorithm: "", crypto.SSECopyKey: "", crypto.SSECopyKeyMD5: ""}, sseRequest: false},                               // 6
-	{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: true},                                                                                        // 6
+	{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: true},                                                                                        // 7
 }
 
 func TestHasServerSideEncryptionHeader(t *testing.T) {
@@ -89,12 +89,12 @@ var hasSSECustomerHeaderTests = []struct {
 	{headers: map[string]string{crypto.SSECKeyMD5: "md5"}, sseRequest: true},                                                                           // 3
 	{headers: map[string]string{}, sseRequest: false},                                                                                                  // 4
 	{headers: map[string]string{crypto.SSECAlgorithm + " ": "AES256", " " + crypto.SSECKey: "key", crypto.SSECKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
-	{headers: map[string]string{crypto.SSECAlgorithm: "", crypto.SSECKey: "", crypto.SSECKeyMD5: ""}, sseRequest: false},                               // 6
+	{headers: map[string]string{crypto.SSECAlgorithm: "", crypto.SSECKey: "", crypto.SSECKeyMD5: ""}, sseRequest: true},                                // 6
 	{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: false},                                                                              // 7
 
 }
 
-func TesthasSSECustomerHeader(t *testing.T) {
+func TestHasSSECustomerHeader(t *testing.T) {
 	for i, test := range hasSSECustomerHeaderTests {
 		headers := http.Header{}
 		for k, v := range test.headers {
@@ -334,6 +334,66 @@ func TestDecryptObjectInfo(t *testing.T) {
 	}
 }
 
+// Tests for issue reproduced when getting the right encrypted
+// offset of the object.
+func TestGetDecryptedRange_Issue50(t *testing.T) {
+	rs, err := parseRequestRangeSpec("bytes=594870256-594870263")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	objInfo := ObjectInfo{
+		Bucket: "bucket",
+		Name:   "object",
+		Size:   595160760,
+		UserDefined: map[string]string{
+			crypto.SSEMultipart:                    "",
+			crypto.SSEIV:                           "HTexa=",
+			crypto.SSESealAlgorithm:                "DAREv2-HMAC-SHA256",
+			crypto.SSECSealedKey:                   "IAA8PGAA==",
+			ReservedMetadataPrefix + "actual-size": "594870264",
+			"content-type":                         "application/octet-stream",
+			"etag":                                 "166b1545b4c1535294ee0686678bea8c-2",
+		},
+		Parts: []objectPartInfo{
+			{
+				Number:     1,
+				Name:       "part.1",
+				ETag:       "etag1",
+				Size:       297580380,
+				ActualSize: 297435132,
+			},
+			{
+				Number:     2,
+				Name:       "part.2",
+				ETag:       "etag2",
+				Size:       297580380,
+				ActualSize: 297435132,
+			},
+		},
+	}
+
+	encOff, encLength, skipLen, seqNumber, partStart, err := objInfo.GetDecryptedRange(rs)
+	if err != nil {
+		t.Fatalf("Test: failed %s", err)
+	}
+	if encOff != 595127964 {
+		t.Fatalf("Test: expected %d, got %d", 595127964, encOff)
+	}
+	if encLength != 32796 {
+		t.Fatalf("Test: expected %d, got %d", 32796, encLength)
+	}
+	if skipLen != 32756 {
+		t.Fatalf("Test: expected %d, got %d", 32756, skipLen)
+	}
+	if seqNumber != 4538 {
+		t.Fatalf("Test: expected %d, got %d", 4538, seqNumber)
+	}
+	if partStart != 1 {
+		t.Fatalf("Test: expected %d, got %d", 1, partStart)
+	}
+}
+
 func TestGetDecryptedRange(t *testing.T) {
 	var (
 		pkgSz     = int64(64) * humanize.KiByte

cmd/endpoint-ellipses_test.go

@@ -287,9 +287,9 @@ func TestParseEndpointSet(t *testing.T) {
 				[]ellipses.ArgPattern{
 					[]ellipses.Pattern{
 						{
-							"/export/set",
-							"",
-							getSequences(1, 64, 0),
+							Prefix: "/export/set",
+							Suffix: "",
+							Seq:    getSequences(1, 64, 0),
 						},
 					},
 				},
@@ -305,14 +305,14 @@ func TestParseEndpointSet(t *testing.T) {
 				[]ellipses.ArgPattern{
 					[]ellipses.Pattern{
 						{
-							"",
-							"",
-							getSequences(1, 64, 0),
+							Prefix: "",
+							Suffix: "",
+							Seq:    getSequences(1, 64, 0),
 						},
 						{
-							"http://minio",
-							"/export/set",
-							getSequences(2, 3, 0),
+							Prefix: "http://minio",
+							Suffix: "/export/set",
+							Seq:    getSequences(2, 3, 0),
 						},
 					},
 				},
@@ -328,9 +328,9 @@ func TestParseEndpointSet(t *testing.T) {
 				[]ellipses.ArgPattern{
 					[]ellipses.Pattern{
 						{
-							"http://minio",
-							".mydomain.net/data",
-							getSequences(1, 64, 0),
+							Prefix: "http://minio",
+							Suffix: ".mydomain.net/data",
+							Seq:    getSequences(1, 64, 0),
 						},
 					},
 				},
@@ -345,14 +345,14 @@ func TestParseEndpointSet(t *testing.T) {
 				[]ellipses.ArgPattern{
 					[]ellipses.Pattern{
 						{
-							"",
-							"/data",
-							getSequences(1, 16, 0),
+							Prefix: "",
+							Suffix: "/data",
+							Seq:    getSequences(1, 16, 0),
 						},
 						{
-							"http://rack",
-							".mydomain.minio",
-							getSequences(1, 4, 0),
+							Prefix: "http://rack",
+							Suffix: ".mydomain.minio",
+							Seq:    getSequences(1, 4, 0),
 						},
 					},
 				},
@@ -368,14 +368,14 @@ func TestParseEndpointSet(t *testing.T) {
 				[]ellipses.ArgPattern{
 					[]ellipses.Pattern{
 						{
-							"",
-							"",
-							getSequences(0, 1, 0),
+							Prefix: "",
+							Suffix: "",
+							Seq:    getSequences(0, 1, 0),
 						},
 						{
-							"http://minio",
-							".mydomain.net/data",
-							getSequences(0, 15, 0),
+							Prefix: "http://minio",
+							Suffix: ".mydomain.net/data",
+							Seq:    getSequences(0, 15, 0),
 						},
 					},
 				},
@@ -391,9 +391,9 @@ func TestParseEndpointSet(t *testing.T) {
 				[]ellipses.ArgPattern{
 					[]ellipses.Pattern{
 						{
-							"http://server1/data",
-							"",
-							getSequences(1, 32, 0),
+							Prefix: "http://server1/data",
+							Suffix: "",
+							Seq:    getSequences(1, 32, 0),
 						},
 					},
 				},
@@ -409,9 +409,9 @@ func TestParseEndpointSet(t *testing.T) {
 				[]ellipses.ArgPattern{
 					[]ellipses.Pattern{
 						{
-							"http://server1/data",
-							"",
-							getSequences(1, 32, 2),
+							Prefix: "http://server1/data",
+							Suffix: "",
+							Seq:    getSequences(1, 32, 2),
 						},
 					},
 				},
@@ -427,19 +427,19 @@ func TestParseEndpointSet(t *testing.T) {
 				[]ellipses.ArgPattern{
 					[]ellipses.Pattern{
 						{
-							"",
-							"",
-							getSequences(1, 2, 0),
+							Prefix: "",
+							Suffix: "",
+							Seq:    getSequences(1, 2, 0),
 						},
 						{
-							"",
-							"/test",
-							getSequences(1, 64, 0),
+							Prefix: "",
+							Suffix: "/test",
+							Seq:    getSequences(1, 64, 0),
 						},
 						{
-							"http://minio",
-							"/export/set",
-							getSequences(2, 3, 0),
+							Prefix: "http://minio",
+							Suffix: "/export/set",
+							Seq:    getSequences(2, 3, 0),
 						},
 					},
 				},
@@ -456,14 +456,14 @@ func TestParseEndpointSet(t *testing.T) {
 				[]ellipses.ArgPattern{
 					[]ellipses.Pattern{
 						{
-							"",
-							"",
-							getSequences(1, 10, 0),
+							Prefix: "",
+							Suffix: "",
+							Seq:    getSequences(1, 10, 0),
 						},
 						{
-							"/export",
-							"/disk",
-							getSequences(1, 10, 0),
+							Prefix: "/export",
+							Suffix: "/disk",
+							Seq:    getSequences(1, 10, 0),
 						},
 					},
 				},

cmd/format-xl.go

@@ -379,6 +379,22 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
 	return disk.RenameFile(minioMetaBucket, formatConfigFileTmp, minioMetaBucket, formatConfigFile)
 }
 
+var ignoredHiddenDirectories = []string{
+	minioMetaBucket,
+	"lost+found",
+	"$RECYCLE.BIN",
+	"System Volume Information",
+}
+
+func isIgnoreHiddenDirectories(dir string) bool {
+	for _, ignDir := range ignoredHiddenDirectories {
+		if dir == ignDir {
+			return true
+		}
+	}
+	return false
+}
+
 // loadFormatXL - loads format.json from disk.
 func loadFormatXL(disk StorageAPI) (format *formatXLV3, err error) {
 	buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
@@ -391,9 +407,7 @@ func loadFormatXL(disk StorageAPI) (format *formatXLV3, err error) {
 			if err != nil {
 				return nil, err
 			}
-			if len(vols) > 1 || (len(vols) == 1 &&
-				vols[0].Name != minioMetaBucket &&
-				vols[0].Name != "lost+found") {
+			if len(vols) > 1 || (len(vols) == 1 && !isIgnoreHiddenDirectories(vols[0].Name)) {
 				// 'format.json' not found, but we
 				// found user data.
 				return nil, errCorruptedFormat

cmd/fs-v1.go

@@ -719,8 +719,12 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 		// Read from fs metadata only if it exists.
 		_, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile)
 		fs.rwPool.Close(fsMetaPath)
-		if rerr != nil && rerr != io.EOF {
-			return oi, rerr
+		if rerr != nil {
+			if rerr != io.EOF {
+				return oi, rerr
+			}
+			// Set Default ETag, if fs.json is empty
+			fsMeta = fs.defaultFsJSON(object)
 		}
 	}
 

cmd/gateway-main.go

@@ -134,9 +134,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Handle common command args.
 	handleCommonCmdArgs(ctx)
 
-	// Handle common env vars.
-	handleCommonEnvVars()
-
 	// Get port to listen on from gateway address
 	_, gatewayPort, pErr := net.SplitHostPort(gatewayAddr)
 	if pErr != nil {
@@ -149,19 +146,26 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// To avoid this error situation we check for port availability.
 	logger.FatalIf(checkPortAvailability(gatewayPort), "Unable to start the gateway")
 
+	// Create certs path.
+	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
+
+	// Check and load TLS certificates.
+	var err error
+	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
+	logger.FatalIf(err, "Invalid TLS certificate file")
+
+	// Check and load Root CAs.
+	globalRootCAs, err = getRootCAs(getCADir())
+	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
+
+	// Handle common env vars.
+	handleCommonEnvVars()
+
 	// Validate if we have access, secret set through environment.
 	if !globalIsEnvCreds {
 		logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
 	}
 
-	// Create certs path.
-	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
-
-	// Check and load SSL certificates.
-	var err error
-	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
-	logger.FatalIf(err, "Invalid SSL certificate file")
-
 	// Set system resources to maximum.
 	logger.LogIf(context.Background(), setMaxResources())
 
@@ -245,9 +249,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(err, "Unable to initialize disk caching")
 	}
 
-	// Load logger subsystem
-	loadLoggers()
-
 	// Re-enable logging
 	logger.Disable = false
 

cmd/gateway-unsupported.go

@@ -30,25 +30,21 @@ type GatewayUnsupported struct{}
 
 // ListMultipartUploads lists all multipart uploads.
 func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return lmi, NotImplemented{}
 }
 
 // NewMultipartUpload upload object in multiple parts
 func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return "", NotImplemented{}
 }
 
 // CopyObjectPart copy part of object to uploadID for another object
 func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, err error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return pi, NotImplemented{}
 }
 
 // PutObjectPart puts a part of object in bucket
 func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, err error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return pi, NotImplemented{}
 }
 
@@ -60,13 +56,11 @@ func (a GatewayUnsupported) ListObjectParts(ctx context.Context, bucket string,
 
 // AbortMultipartUpload aborts a ongoing multipart upload
 func (a GatewayUnsupported) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
-	logger.LogIf(ctx, NotImplemented{})
 	return NotImplemented{}
 }
 
 // CompleteMultipartUpload completes ongoing multipart upload and finalizes object
 func (a GatewayUnsupported) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return oi, NotImplemented{}
 }
 
@@ -78,13 +72,11 @@ func (a GatewayUnsupported) SetBucketPolicy(ctx context.Context, bucket string,
 
 // GetBucketPolicy will get policy on bucket
 func (a GatewayUnsupported) GetBucketPolicy(ctx context.Context, bucket string) (bucketPolicy *policy.Policy, err error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return nil, NotImplemented{}
 }
 
 // DeleteBucketPolicy deletes all policies on bucket
 func (a GatewayUnsupported) DeleteBucketPolicy(ctx context.Context, bucket string) error {
-	logger.LogIf(ctx, NotImplemented{})
 	return NotImplemented{}
 }
 
@@ -95,50 +87,42 @@ func (a GatewayUnsupported) ReloadFormat(ctx context.Context, dryRun bool) error
 
 // HealFormat - Not implemented stub
 func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return madmin.HealResultItem{}, NotImplemented{}
 }
 
 // HealBucket - Not implemented stub
 func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return nil, NotImplemented{}
 }
 
 // ListBucketsHeal - Not implemented stub
 func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return nil, NotImplemented{}
 }
 
 // HealObject - Not implemented stub
 func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return h, NotImplemented{}
 }
 
 // ListObjectsV2 - Not implemented stub
 func (a GatewayUnsupported) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return result, NotImplemented{}
 }
 
 // ListObjectsHeal - Not implemented stub
 func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return loi, NotImplemented{}
 }
 
 // CopyObject copies a blob from source container to destination container.
 func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
 	srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
-	logger.LogIf(ctx, NotImplemented{})
 	return objInfo, NotImplemented{}
 }
 
 // RefreshBucketPolicy refreshes cache policy with what's on disk.
 func (a GatewayUnsupported) RefreshBucketPolicy(ctx context.Context, bucket string) error {
-	logger.LogIf(ctx, NotImplemented{})
 	return NotImplemented{}
 }
 

cmd/gateway/azure/gateway-azure.go

@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/go-autorest/autorest/azure"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
@@ -93,7 +94,7 @@ EXAMPLES:
   2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
      $ export MINIO_ACCESS_KEY=azureaccountname
      $ export MINIO_SECRET_KEY=azureaccountkey
-     $ {{.HelpName}} https://azure.example.com
+     $ {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint
 
   3. Start minio gateway server for Azure Blob Storage backend with edge caching enabled.
      $ export MINIO_ACCESS_KEY=azureaccountname
@@ -140,28 +141,43 @@ func (g *Azure) Name() string {
 	return azureBackend
 }
 
+// All known cloud environments of Azure
+var azureEnvs = []azure.Environment{
+	azure.PublicCloud,
+	azure.USGovernmentCloud,
+	azure.ChinaCloud,
+	azure.GermanCloud,
+}
+
 // NewGatewayLayer initializes azure blob storage client and returns AzureObjects.
 func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
 	var err error
-	var endpoint = storage.DefaultBaseURL
+	// The default endpoint is the public cloud
+	var endpoint = azure.PublicCloud.StorageEndpointSuffix
 	var secure = true
 
-	// If user provided some parameters
+	// Load the endpoint url if supplied by the user.
 	if g.host != "" {
 		endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
 		if err != nil {
 			return nil, err
 		}
+		// Reformat the full account storage endpoint to the base format.
+		//   e.g. testazure.blob.core.windows.net => core.windows.net
+		endpoint = strings.ToLower(endpoint)
+		for _, env := range azureEnvs {
+			if strings.Contains(endpoint, env.StorageEndpointSuffix) {
+				endpoint = env.StorageEndpointSuffix
+				break
+			}
+		}
 	}
 
-	if endpoint == fmt.Sprintf("%s.blob.%s", creds.AccessKey, storage.DefaultBaseURL) {
-		// If, by mistake, user provides endpoint as accountname.blob.core.windows.net
-		endpoint = storage.DefaultBaseURL
-	}
 	c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure)
 	if err != nil {
 		return &azureObjects{}, err
 	}
+
 	c.AddToUserAgent(fmt.Sprintf("APN/1.0 Minio/1.0 Minio/%s", minio.Version))
 	c.HTTPClient = &http.Client{Transport: minio.NewCustomHTTPTransport()}
 
@@ -672,7 +688,6 @@ func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, sta
 	}
 	_, err = io.Copy(writer, rc)
 	rc.Close()
-	logger.LogIf(ctx, err)
 	return err
 }
 

cmd/gateway/s3/gateway-s3.go

@@ -203,6 +203,9 @@ func newS3(url string) (*miniogo.Core, error) {
 		return nil, err
 	}
 
+	// Set custom transport
+	clnt.SetCustomTransport(minio.NewCustomHTTPTransport())
+
 	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
 	// Check if the provided keys are valid.
 	if _, err = clnt.BucketExists(probeBucketName); err != nil {
@@ -341,13 +344,13 @@ func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 	var objInfo minio.ObjectInfo
 	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
-		return nil, err
+		return nil, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
 	var startOffset, length int64
 	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
 	if err != nil {
-		return nil, err
+		return nil, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
 	pr, pw := io.Pipe()
@@ -377,7 +380,6 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 
 	if startOffset >= 0 && length >= 0 {
 		if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
-			logger.LogIf(ctx, err)
 			return minio.ErrorRespToObjectError(err, bucket, key)
 		}
 	}
@@ -388,7 +390,6 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 	defer object.Close()
 
 	if _, err := io.Copy(writer, object); err != nil {
-		logger.LogIf(ctx, err)
 		return minio.ErrorRespToObjectError(err, bucket, key)
 	}
 	return nil
@@ -396,7 +397,7 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 
 // GetObjectInfo reads object info and replies back ObjectInfo
 func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{miniogo.GetObjectOptions{ServerSideEncryption: opts.ServerSideEncryption}})
+	oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{GetObjectOptions: miniogo.GetObjectOptions{ServerSideEncryption: opts.ServerSideEncryption}})
 	if err != nil {
 		return minio.ObjectInfo{}, minio.ErrorRespToObjectError(err, bucket, object)
 	}
@@ -494,7 +495,7 @@ func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, de
 func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, e error) {
 	result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
-		return lpi, err
+		return lpi, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
 	return minio.FromMinioClientListPartsInfo(result), nil

cmd/generic-handlers.go

@@ -655,7 +655,7 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		if err == dns.ErrNoEntriesFound {
 			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
 		} else {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL)
 		}
 		return
 	}
@@ -783,7 +783,11 @@ type sseTLSHandler struct{ handler http.Handler }
 func (h sseTLSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Deny SSE-C requests if not made over TLS
 	if !globalIsSSL && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
-		writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
+		if r.Method == http.MethodHead {
+			writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
+		} else {
+			writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL)
+		}
 		return
 	}
 	h.handler.ServeHTTP(w, r)

cmd/generic-handlers_test.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"strconv"
 	"testing"
 
@@ -184,14 +185,15 @@ func TestContainsReservedMetadata(t *testing.T) {
 }
 
 var sseTLSHandlerTests = []struct {
+	URL               *url.URL
 	Header            http.Header
 	IsTLS, ShouldFail bool
 }{
-	{Header: http.Header{}, IsTLS: false, ShouldFail: false},                                        // 0
-	{Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: false, ShouldFail: true}, // 1
-	{Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: true, ShouldFail: false}, // 2
-	{Header: http.Header{crypto.SSECKey: []string{""}}, IsTLS: true, ShouldFail: false},             // 3
-	{Header: http.Header{crypto.SSECopyAlgorithm: []string{""}}, IsTLS: false, ShouldFail: true},    // 4
+	{URL: &url.URL{}, Header: http.Header{}, IsTLS: false, ShouldFail: false},                                        // 0
+	{URL: &url.URL{}, Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: false, ShouldFail: true}, // 1
+	{URL: &url.URL{}, Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: true, ShouldFail: false}, // 2
+	{URL: &url.URL{}, Header: http.Header{crypto.SSECKey: []string{""}}, IsTLS: true, ShouldFail: false},             // 3
+	{URL: &url.URL{}, Header: http.Header{crypto.SSECopyAlgorithm: []string{""}}, IsTLS: false, ShouldFail: true},    // 4
 }
 
 func TestSSETLSHandler(t *testing.T) {
@@ -206,6 +208,7 @@ func TestSSETLSHandler(t *testing.T) {
 		w := httptest.NewRecorder()
 		r := new(http.Request)
 		r.Header = test.Header
+		r.URL = test.URL
 
 		h := setSSETLSHandler(okHandler)
 		h.ServeHTTP(w, r)

cmd/handler-utils.go

@@ -176,14 +176,26 @@ func getRedirectPostRawQuery(objInfo ObjectInfo) string {
 	return redirectValues.Encode()
 }
 
+// Returns access key in the request Authorization header.
+func getReqAccessKey(r *http.Request, region string) (accessKey string) {
+	cred, _, _ := getReqAccessKeyV4(r, region)
+	if cred.AccessKey == "" {
+		cred, _, _ = getReqAccessKeyV2(r)
+	}
+	return cred.AccessKey
+}
+
 // Extract request params to be sent with event notifiation.
 func extractReqParams(r *http.Request) map[string]string {
 	if r == nil {
 		return nil
 	}
 
+	region := globalServerConfig.GetRegion()
 	// Success.
 	return map[string]string{
+		"region":          region,
+		"accessKey":       getReqAccessKey(r, region),
 		"sourceIPAddress": handlers.GetSourceIP(r),
 		// Add more fields here.
 	}
@@ -193,6 +205,7 @@ func extractReqParams(r *http.Request) map[string]string {
 func extractRespElements(w http.ResponseWriter) map[string]string {
 
 	return map[string]string{
+		"requestId":      w.Header().Get(responseRequestIDKey),
 		"content-length": w.Header().Get("Content-Length"),
 		// Add more fields here.
 	}

cmd/handler-utils_test.go

@@ -172,6 +172,16 @@ func TestExtractMetadataHeaders(t *testing.T) {
 			},
 			shouldFail: false,
 		},
+		// Support multiple values
+		{
+			header: http.Header{
+				"x-amz-meta-key": []string{"amz-meta1", "amz-meta2"},
+			},
+			metadata: map[string]string{
+				"x-amz-meta-key": "amz-meta1,amz-meta2",
+			},
+			shouldFail: false,
+		},
 		// Empty header input returns empty metadata.
 		{
 			header:     nil,

cmd/http/listener.go

@@ -141,8 +141,8 @@ func getMethodResourceHost(bufConn *BufConn, maxHeaderBytes int) (method string,
 			// HTTP headers are case insensitive, so we should simply convert
 			// each tokens to their lower case form to match 'host' header.
 			token = strings.ToLower(token)
-			if strings.HasPrefix(token, "host: ") {
-				host = strings.TrimPrefix(strings.TrimSuffix(token, "\r"), "host: ")
+			if strings.HasPrefix(token, "host:") {
+				host = strings.TrimPrefix(strings.TrimSuffix(token, "\r"), "host:")
 				return method, resource, host, nil
 			}
 		}

cmd/http/listener_test.go

@@ -116,7 +116,7 @@ func getTLSConfig(t *testing.T) *tls.Config {
 	tlsConfig := &tls.Config{
 		PreferServerCipherSuites: true,
 		MinVersion:               tls.VersionTLS12,
-		NextProtos:               []string{"http/1.1", "h2"},
+		NextProtos:               []string{"http/1.1"},
 	}
 	tlsConfig.Certificates = append(tlsConfig.Certificates, tlsCert)
 

cmd/http/server.go

@@ -192,7 +192,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			CipherSuites:             defaultCipherSuites,
 			CurvePreferences:         secureCurves,
 			MinVersion:               tls.VersionTLS12,
-			NextProtos:               []string{"http/1.1", "h2"},
+			NextProtos:               []string{"http/1.1"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/iam.go

@@ -269,12 +269,46 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 }
 
 // SetTempUser - set temporary user credentials, these credentials have an expiry.
-func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials) error {
+func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyName string) error {
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
+	sys.Lock()
+	defer sys.Unlock()
+
+	// If OPA is not set we honor any policy claims for this
+	// temporary user which match with pre-configured canned
+	// policies for this server.
+	if globalPolicyOPA == nil && policyName != "" {
+		p, ok := sys.iamCannedPolicyMap[policyName]
+		if !ok {
+			return errInvalidArgument
+		}
+		if p.IsEmpty() {
+			delete(sys.iamPolicyMap, accessKey)
+			return nil
+		}
+
+		data, err := json.Marshal(policyName)
+		if err != nil {
+			return err
+		}
+
+		configFile := pathJoin(iamConfigSTSPrefix, accessKey, iamPolicyFile)
+		if globalEtcdClient != nil {
+			err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+		} else {
+			err = saveConfig(context.Background(), objectAPI, configFile, data)
+		}
+		if err != nil {
+			return err
+		}
+
+		sys.iamPolicyMap[accessKey] = policyName
+	}
+
 	configFile := pathJoin(iamConfigSTSPrefix, accessKey, iamIdentityFile)
 	data, err := json.Marshal(cred)
 	if err != nil {
@@ -291,9 +325,6 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials) error {
 		return err
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
-
 	sys.iamUsersMap[accessKey] = cred
 	return nil
 }

cmd/jwt.go

@@ -110,34 +110,6 @@ func authenticateURL(accessKey, secretKey string) (string, error) {
 	return authenticateJWTUsers(accessKey, secretKey, defaultURLJWTExpiry)
 }
 
-func stsTokenCallback(jwtToken *jwtgo.Token) (interface{}, error) {
-	if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
-		return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
-	}
-
-	if err := jwtToken.Claims.Valid(); err != nil {
-		return nil, errAuthentication
-	}
-	if claims, ok := jwtToken.Claims.(jwtgo.MapClaims); ok {
-		accessKey, ok := claims["accessKey"].(string)
-		if !ok {
-			return nil, errInvalidAccessKeyID
-		}
-		if accessKey == globalServerConfig.GetCredential().AccessKey {
-			return []byte(globalServerConfig.GetCredential().SecretKey), nil
-		}
-		if globalIAMSys == nil {
-			return nil, errInvalidAccessKeyID
-		}
-		_, ok = globalIAMSys.GetUser(accessKey)
-		if !ok {
-			return nil, errInvalidAccessKeyID
-		}
-		return []byte(globalServerConfig.GetCredential().SecretKey), nil
-	}
-	return nil, errAuthentication
-}
-
 // Callback function used for parsing
 func webTokenCallback(jwtToken *jwtgo.Token) (interface{}, error) {
 	if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {

cmd/logger/audit-logger.go

@@ -0,0 +1,97 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Represents the current version of audit log structure.
+const auditLogVersion = "1"
+
+// AuditEntry - audit entry logs.
+type AuditEntry struct {
+	Version      string            `json:"version"`
+	DeploymentID string            `json:"deploymentid,omitempty"`
+	Time         string            `json:"time"`
+	API          *api              `json:"api,omitempty"`
+	RemoteHost   string            `json:"remotehost,omitempty"`
+	RequestID    string            `json:"requestID,omitempty"`
+	UserAgent    string            `json:"userAgent,omitempty"`
+	ReqQuery     map[string]string `json:"requestQuery,omitempty"`
+	ReqHeader    map[string]string `json:"requestHeader,omitempty"`
+	RespHeader   map[string]string `json:"responseHeader,omitempty"`
+}
+
+// AuditTargets is the list of enabled audit loggers
+var AuditTargets = []LoggingTarget{}
+
+// AddAuditTarget adds a new audit logger target to the
+// list of enabled loggers
+func AddAuditTarget(t LoggingTarget) {
+	AuditTargets = append(AuditTargets, t)
+}
+
+// AuditLog - logs audit logs to all targets.
+func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if Disable {
+		return
+	}
+
+	req := GetReqInfo(ctx)
+	if req == nil {
+		req = &ReqInfo{API: "SYSTEM"}
+	}
+
+	reqQuery := make(map[string]string)
+	for k, v := range r.URL.Query() {
+		reqQuery[k] = strings.Join(v, ",")
+	}
+	reqHeader := make(map[string]string)
+	for k, v := range r.Header {
+		reqHeader[k] = strings.Join(v, ",")
+	}
+	respHeader := make(map[string]string)
+	for k, v := range w.Header() {
+		respHeader[k] = strings.Join(v, ",")
+	}
+
+	// Send audit logs only to http targets.
+	for _, t := range AuditTargets {
+		t.send(AuditEntry{
+			Version:      auditLogVersion,
+			DeploymentID: deploymentID,
+			RemoteHost:   req.RemoteHost,
+			RequestID:    req.RequestID,
+			UserAgent:    req.UserAgent,
+			Time:         time.Now().UTC().Format(time.RFC3339Nano),
+			API: &api{
+				Name: req.API,
+				Args: &args{
+					Bucket: req.BucketName,
+					Object: req.ObjectName,
+				},
+			},
+			ReqQuery:   reqQuery,
+			ReqHeader:  reqHeader,
+			RespHeader: respHeader,
+		})
+	}
+}

cmd/logger/logger.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"go/build"
-	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -117,9 +116,11 @@ type traceEntry struct {
 	Source    []string          `json:"source,omitempty"`
 	Variables map[string]string `json:"variables,omitempty"`
 }
+
 type args struct {
-	Bucket string `json:"bucket,omitempty"`
-	Object string `json:"object,omitempty"`
+	Bucket   string            `json:"bucket,omitempty"`
+	Object   string            `json:"object,omitempty"`
+	Metadata map[string]string `json:"metadata,omitempty"`
 }
 
 type api struct {
@@ -320,7 +321,7 @@ func logIf(ctx context.Context, err error) {
 	}
 
 	// Get full stack trace
-	trace := getTrace(2)
+	trace := getTrace(3)
 
 	// Get the cause for the Error
 	message := err.Error()
@@ -342,55 +343,6 @@ func logIf(ctx context.Context, err error) {
 	}
 }
 
-type auditEntry struct {
-	DeploymentID string            `json:"deploymentid,omitempty"`
-	Time         string            `json:"time"`
-	API          *api              `json:"api,omitempty"`
-	RemoteHost   string            `json:"remotehost,omitempty"`
-	RequestID    string            `json:"requestID,omitempty"`
-	UserAgent    string            `json:"userAgent,omitempty"`
-	Metadata     map[string]string `json:"metadata,omitempty"`
-}
-
-// AuditLog - logs audit logs to all targets.
-func AuditLog(ctx context.Context, r *http.Request) {
-	if Disable {
-		return
-	}
-
-	req := GetReqInfo(ctx)
-	if req == nil {
-		req = &ReqInfo{API: "SYSTEM"}
-	}
-
-	API := "SYSTEM"
-	if req.API != "" {
-		API = req.API
-	}
-
-	tags := make(map[string]string)
-	for _, entry := range req.GetTags() {
-		tags[entry.Key] = entry.Val
-	}
-
-	entry := auditEntry{
-		DeploymentID: deploymentID,
-		RemoteHost:   req.RemoteHost,
-		RequestID:    req.RequestID,
-		UserAgent:    req.UserAgent,
-		Time:         time.Now().UTC().Format(time.RFC3339Nano),
-		API:          &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
-		Metadata:     tags,
-	}
-
-	// Send audit logs only to http targets.
-	for _, t := range Targets {
-		if _, ok := t.(*HTTPTarget); ok {
-			t.send(entry)
-		}
-	}
-}
-
 // ErrCritical is the value panic'd whenever CriticalIf is called.
 var ErrCritical struct{}
 

cmd/naughty-disk_test.go

@@ -159,6 +159,13 @@ func (d *naughtyDisk) DeleteFile(volume string, path string) (err error) {
 	return d.disk.DeleteFile(volume, path)
 }
 
+func (d *naughtyDisk) WriteAll(volume string, path string, buf []byte) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.WriteAll(volume, path, buf)
+}
+
 func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error) {
 	if err := d.calcError(); err != nil {
 		return nil, err

cmd/notification.go

@@ -494,12 +494,11 @@ func (args eventArgs) ToEvent() event.Event {
 		return fmt.Sprintf("%s://%s:%s", getURLScheme(globalIsSSL), host, globalMinioPort)
 	}
 
-	creds := globalServerConfig.GetCredential()
 	eventTime := UTCNow()
 	uniqueID := fmt.Sprintf("%X", eventTime.UnixNano())
 
 	respElements := map[string]string{
-		"x-amz-request-id":        uniqueID,
+		"x-amz-request-id":        args.RespElements["requestId"],
 		"x-minio-origin-endpoint": getOriginEndpoint(), // Minio specific custom elements.
 	}
 	if args.RespElements["content-length"] != "" {
@@ -508,10 +507,10 @@ func (args eventArgs) ToEvent() event.Event {
 	newEvent := event.Event{
 		EventVersion:      "2.0",
 		EventSource:       "minio:s3",
-		AwsRegion:         globalServerConfig.GetRegion(),
+		AwsRegion:         args.ReqParams["region"],
 		EventTime:         eventTime.Format(event.AMZTimeFormat),
 		EventName:         args.EventName,
-		UserIdentity:      event.Identity{creds.AccessKey},
+		UserIdentity:      event.Identity{PrincipalID: args.ReqParams["accessKey"]},
 		RequestParameters: args.ReqParams,
 		ResponseElements:  respElements,
 		S3: event.Metadata{
@@ -519,7 +518,7 @@ func (args eventArgs) ToEvent() event.Event {
 			ConfigurationID: "Config",
 			Bucket: event.Bucket{
 				Name:          args.BucketName,
-				OwnerIdentity: event.Identity{creds.AccessKey},
+				OwnerIdentity: event.Identity{PrincipalID: args.ReqParams["accessKey"]},
 				ARN:           policy.ResourceARNPrefix + args.BucketName,
 			},
 			Object: event.Object{

cmd/object-api-multipart_test.go

@@ -329,20 +329,20 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 		// Test case - 12.
 		// Input to replicate Md5 mismatch.
 		{bucket, object, uploadID, 1, "", "d41d8cd98f00b204e9800998ecf8427f", "", 0, false, "",
-			hash.BadDigest{"d41d8cd98f00b204e9800998ecf8427f", "d41d8cd98f00b204e9800998ecf8427e"}},
+			hash.BadDigest{ExpectedMD5: "d41d8cd98f00b204e9800998ecf8427f", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"}},
 		// Test case - 13.
 		// When incorrect sha256 is provided.
 		{bucket, object, uploadID, 1, "", "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b854", 0, false, "",
-			hash.SHA256Mismatch{"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b854",
-				"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}},
+			hash.SHA256Mismatch{ExpectedSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b854",
+				CalculatedSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}},
 		// Test case - 14.
 		// Input with size more than the size of actual data inside the reader.
 		{bucket, object, uploadID, 1, "abcd", "e2fc714c4727ee9395f324cd2e7f3335", "", int64(len("abcd") + 1), false, "",
-			hash.BadDigest{"e2fc714c4727ee9395f324cd2e7f3335", "e2fc714c4727ee9395f324cd2e7f331f"}},
+			hash.BadDigest{ExpectedMD5: "e2fc714c4727ee9395f324cd2e7f3335", CalculatedMD5: "e2fc714c4727ee9395f324cd2e7f331f"}},
 		// Test case - 15.
 		// Input with size less than the size of actual data inside the reader.
 		{bucket, object, uploadID, 1, "abcd", "900150983cd24fb0d6963f7d28e17f73", "", int64(len("abcd") - 1), false, "",
-			hash.BadDigest{"900150983cd24fb0d6963f7d28e17f73", "900150983cd24fb0d6963f7d28e17f72"}},
+			hash.BadDigest{ExpectedMD5: "900150983cd24fb0d6963f7d28e17f73", CalculatedMD5: "900150983cd24fb0d6963f7d28e17f72"}},
 
 		// Test case - 16-19.
 		// Validating for success cases.

cmd/object-api-putobject_test.go

@@ -97,24 +97,24 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 		// Test case - 7.
 		// Input to replicate Md5 mismatch.
 		{bucket, object, []byte(""), map[string]string{"etag": "d41d8cd98f00b204e9800998ecf8427f"}, "", 0, "",
-			hash.BadDigest{"d41d8cd98f00b204e9800998ecf8427f", "d41d8cd98f00b204e9800998ecf8427e"}},
+			hash.BadDigest{ExpectedMD5: "d41d8cd98f00b204e9800998ecf8427f", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"}},
 
 		// Test case - 8.
 		// With incorrect sha256.
 		{bucket, object, []byte("abcd"), map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331f"},
 			"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580", int64(len("abcd")),
-			"", hash.SHA256Mismatch{"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
-				"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589"}},
+			"", hash.SHA256Mismatch{ExpectedSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580",
+				CalculatedSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589"}},
 
 		// Test case - 9.
 		// Input with size more than the size of actual data inside the reader.
 		{bucket, object, []byte("abcd"), map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331e"}, "", int64(len("abcd") + 1), "",
-			hash.BadDigest{"e2fc714c4727ee9395f324cd2e7f331e", "e2fc714c4727ee9395f324cd2e7f331f"}},
+			hash.BadDigest{ExpectedMD5: "e2fc714c4727ee9395f324cd2e7f331e", CalculatedMD5: "e2fc714c4727ee9395f324cd2e7f331f"}},
 
 		// Test case - 10.
 		// Input with size less than the size of actual data inside the reader.
 		{bucket, object, []byte("abcd"), map[string]string{"etag": "900150983cd24fb0d6963f7d28e17f73"}, "", int64(len("abcd") - 1), "",
-			hash.BadDigest{"900150983cd24fb0d6963f7d28e17f73", "900150983cd24fb0d6963f7d28e17f72"}},
+			hash.BadDigest{ExpectedMD5: "900150983cd24fb0d6963f7d28e17f73", CalculatedMD5: "900150983cd24fb0d6963f7d28e17f72"}},
 
 		// Test case - 11-14.
 		// Validating for success cases.
@@ -144,11 +144,11 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 		// Test case 24-26.
 		// data with invalid md5sum in header
 		{bucket, object, data, invalidMD5Header, "", int64(len(data)), getMD5Hash(data),
-			hash.BadDigest{invalidMD5, getMD5Hash(data)}},
+			hash.BadDigest{ExpectedMD5: invalidMD5, CalculatedMD5: getMD5Hash(data)}},
 		{bucket, object, nilBytes, invalidMD5Header, "", int64(len(nilBytes)), getMD5Hash(nilBytes),
-			hash.BadDigest{invalidMD5, getMD5Hash(nilBytes)}},
+			hash.BadDigest{ExpectedMD5: invalidMD5, CalculatedMD5: getMD5Hash(nilBytes)}},
 		{bucket, object, fiveMBBytes, invalidMD5Header, "", int64(len(fiveMBBytes)), getMD5Hash(fiveMBBytes),
-			hash.BadDigest{invalidMD5, getMD5Hash(fiveMBBytes)}},
+			hash.BadDigest{ExpectedMD5: invalidMD5, CalculatedMD5: getMD5Hash(fiveMBBytes)}},
 
 		// Test case 27-29.
 		// data with size different from the actual number of bytes available in the reader

cmd/object-handlers.go

@@ -33,6 +33,7 @@ import (
 
 	snappy "github.com/golang/snappy"
 	"github.com/gorilla/mux"
+	"github.com/klauspost/readahead"
 	miniogo "github.com/minio/minio-go"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
@@ -78,7 +79,7 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
 func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SelectObject")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	// Fetch object stat info.
 	objectAPI := api.ObjectAPI()
@@ -123,7 +124,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 				}
 
 				_, err := getObjectInfo(ctx, bucket, object, ObjectOptions{})
-				if toAPIErrorCode(err) == ErrNoSuchKey {
+				if toAPIErrorCode(ctx, err) == ErrNoSuchKey {
 					s3Error = ErrNoSuchKey
 				}
 			}
@@ -165,7 +166,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 
 	gr, err := getObjectNInfo(ctx, bucket, object, nil, r.Header, readLock, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	defer gr.Close()
@@ -229,9 +230,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 
 	}
 	if selectReq.InputSerialization.JSON != nil {
-		if selectReq.InputSerialization.JSON.Type != s3select.JSONTypeDocument &&
-			selectReq.InputSerialization.JSON.Type != s3select.JSONLinesType &&
-			selectReq.InputSerialization.JSON.Type != "" {
+		if selectReq.InputSerialization.JSON.Type != s3select.JSONLinesType {
 			writeErrorResponse(w, ErrInvalidJSONType, r.URL)
 			return
 		}
@@ -251,30 +250,51 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		}
 	}
 
-	s3s, err := s3select.New(gr, objInfo.Size, selectReq)
+	reader := readahead.NewReader(gr)
+	defer reader.Close()
+
+	size := objInfo.Size
+	if objInfo.IsCompressed() {
+		size = objInfo.GetActualSize()
+		if size < 0 {
+			writeErrorResponse(w, toAPIErrorCode(ctx, errInvalidDecompressedSize), r.URL)
+			return
+		}
+	}
+
+	s3s, err := s3select.New(reader, size, selectReq)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
 	// Parses the select query and checks for an error
 	_, _, _, _, _, _, err = s3select.ParseSelect(s3s)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
 	// Executes the query on data-set
-	if err = s3select.Execute(w, s3s); err != nil {
-		logger.LogIf(ctx, err)
+	s3select.Execute(w, s3s)
 
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
+	if err != nil {
+		host, port = "", ""
 	}
 
-	for k, v := range objInfo.UserDefined {
-		logger.GetReqInfo(ctx).SetTags(k, v)
-	}
-
-	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
+	// Notify object accessed via a GET request.
+	sendEvent(eventArgs{
+		EventName:    event.ObjectAccessedGet,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
+	})
 }
 
 // GetObjectHandler - GET Object
@@ -284,7 +304,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObject")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -330,7 +350,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				_, err := getObjectInfo(ctx, bucket, object, opts)
-				if toAPIErrorCode(err) == ErrNoSuchKey {
+				if toAPIErrorCode(ctx, err) == ErrNoSuchKey {
 					s3Error = ErrNoSuchKey
 				}
 			}
@@ -364,15 +384,16 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 
 	gr, err := getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, opts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	defer gr.Close()
+
 	objInfo := gr.ObjInfo
 
 	if objectAPI.IsEncryptionSupported() {
 		if _, err = DecryptObjectInfo(objInfo, r.Header); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 	}
@@ -395,8 +416,8 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	if hErr := setObjectHeaders(w, objInfo, rs); hErr != nil {
-		writeErrorResponse(w, toAPIErrorCode(hErr), r.URL)
+	if err = setObjectHeaders(w, objInfo, rs); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -412,14 +433,14 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 	// Write object content to response body
 	if _, err = io.Copy(httpWriter, gr); err != nil {
 		if !httpWriter.HasWritten() && !statusCodeWritten { // write error response only if no data or headers has been written to client yet
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		}
 		return
 	}
 
 	if err = httpWriter.Close(); err != nil {
 		if !httpWriter.HasWritten() && !statusCodeWritten { // write error response only if no data or headers has been written to client yet
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 	}
@@ -441,12 +462,6 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		Host:         host,
 		Port:         port,
 	})
-
-	for k, v := range objInfo.UserDefined {
-		logger.GetReqInfo(ctx).SetTags(k, v)
-	}
-
-	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // HeadObjectHandler - HEAD Object
@@ -455,7 +470,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadObject")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -499,7 +514,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 				IsOwner:         false,
 			}) {
 				_, err := getObjectInfo(ctx, bucket, object, opts)
-				if toAPIErrorCode(err) == ErrNoSuchKey {
+				if toAPIErrorCode(ctx, err) == ErrNoSuchKey {
 					s3Error = ErrNoSuchKey
 				}
 			}
@@ -528,13 +543,13 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
-		writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
+		writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
 		return
 	}
 
 	if objectAPI.IsEncryptionSupported() {
 		if _, err = DecryptObjectInfo(objInfo, r.Header); err != nil {
-			writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
+			writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
 			return
 		}
 	}
@@ -548,7 +563,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 			case crypto.SSEC.IsEncrypted(objInfo.UserDefined):
 				// Validate the SSE-C Key set in the header.
 				if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil {
-					writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
+					writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
 					return
 				}
 				w.Header().Set(crypto.SSECAlgorithm, r.Header.Get(crypto.SSECAlgorithm))
@@ -563,8 +578,8 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	// Set standard object headers.
-	if hErr := setObjectHeaders(w, objInfo, rs); hErr != nil {
-		writeErrorResponseHeadersOnly(w, toAPIErrorCode(hErr))
+	if err = setObjectHeaders(w, objInfo, rs); err != nil {
+		writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
 		return
 	}
 
@@ -595,12 +610,6 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		Host:         host,
 		Port:         port,
 	})
-
-	for k, v := range objInfo.UserDefined {
-		logger.GetReqInfo(ctx).SetTags(k, v)
-	}
-
-	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // Extract metadata relevant for an CopyObject operation based on conditional
@@ -641,7 +650,7 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
 func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObject")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -678,6 +687,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	// Check if metadata directive is valid.
 	if !isMetadataDirectiveValid(r.Header) {
 		writeErrorResponse(w, ErrInvalidMetadataDirective, r.URL)
@@ -701,33 +715,15 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		getObjectNInfo = api.CacheAPI().GetObjectNInfo
 	}
 
-	// Get request range.
-	var rs *HTTPRangeSpec
-	rangeHeader := r.Header.Get("x-amz-copy-source-range")
-	if rangeHeader != "" {
-		var parseRangeErr error
-		if rs, parseRangeErr = parseRequestRangeSpec(rangeHeader); parseRangeErr != nil {
-			// Handle only errInvalidRange. Ignore other
-			// parse error and treat it as regular Get
-			// request like Amazon S3.
-			if parseRangeErr == errInvalidRange {
-				writeErrorResponse(w, ErrInvalidRange, r.URL)
-				return
-			}
-
-			// log the error.
-			logger.LogIf(ctx, parseRangeErr)
-		}
-	}
-
 	var lock = noLock
 	if !cpSrcDstSame {
 		lock = readLock
 	}
 
+	var rs *HTTPRangeSpec
 	gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, lock, srcOpts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	defer gr.Close()
@@ -769,7 +765,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	if crypto.IsEncrypted(srcInfo.UserDefined) {
 		actualSize, err = srcInfo.DecryptedSize()
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 	}
@@ -808,7 +804,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualSize)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -816,7 +812,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	if objectAPI.IsEncryptionSupported() && !isCompressed {
 		// Encryption parameters not applicable for this object.
 		if !crypto.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header) {
-			writeErrorResponse(w, toAPIErrorCode(errInvalidEncryptionParameters), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, errInvalidEncryptionParameters), r.URL)
 			return
 		}
 		// Encryption parameters not present for this object.
@@ -836,25 +832,30 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		if sseC {
 			newKey, err = ParseSSECustomerRequest(r)
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}
-		// AWS S3 implementation requires us to only rotate keys
-		// when/ both keys are provided and destination is same
-		// otherwise we proceed to encrypt/decrypt.
-		if sseCopyC && sseC && cpSrcDstSame {
-			// Get the old key which needs to be rotated.
-			oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined)
-			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-				return
+
+		// If src == dst and either
+		// - the object is encrypted using SSE-C and two different SSE-C keys are present
+		// - the object is encrypted using SSE-S3 and the SSE-S3 header is present
+		// than execute a key rotation.
+		if cpSrcDstSame && ((sseCopyC && sseC) || (sseS3 && sseCopyS3)) {
+			if sseCopyC && sseC {
+				oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined)
+				if err != nil {
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+					return
+				}
 			}
 			for k, v := range srcInfo.UserDefined {
 				encMetadata[k] = v
 			}
+
+			// In case of SSE-S3 oldKey and newKey aren't used - the KMS manages the keys.
 			if err = rotateKey(oldKey, newKey, srcBucket, srcObject, encMetadata); err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 
@@ -886,7 +887,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			if isTargetEncrypted {
 				reader, err = newEncryptReader(reader, newKey, dstBucket, dstObject, encMetadata, sseS3)
 				if err != nil {
-					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 					return
 				}
 			}
@@ -899,7 +900,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 			srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", targetSize) // do not try to verify encrypted content
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}
@@ -925,7 +926,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	// metadataOnly is true indicating that we are not overwriting the object.
 	// if encryption is enabled we do not need explicit "REPLACE" metadata to
 	// be enabled as well - this is to allow for key-rotation.
-	if !isMetadataReplace(r.Header) && srcInfo.metadataOnly && !crypto.SSEC.IsEncrypted(srcInfo.UserDefined) {
+	if !isMetadataReplace(r.Header) && srcInfo.metadataOnly && !crypto.IsEncrypted(srcInfo.UserDefined) {
 		// If x-amz-metadata-directive is not set to REPLACE then we need
 		// to error out if source and destination are same.
 		writeErrorResponse(w, ErrInvalidCopyDest, r.URL)
@@ -972,7 +973,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		// object is same then only metadata is updated.
 		objInfo, err = objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 	}
@@ -995,20 +996,15 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// Notify object created event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedCopy,
-		BucketName: dstBucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedCopy,
+		BucketName:   dstBucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
-
-	for k, v := range objInfo.UserDefined {
-		logger.GetReqInfo(ctx).SetTags(k, v)
-	}
-
-	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // PutObjectHandler - PUT Object
@@ -1022,7 +1018,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObject")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -1070,7 +1066,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			}
 			size, err = strconv.ParseInt(sizeStr[0], 10, 64)
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}
@@ -1165,7 +1161,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		var actualReader *hash.Reader
 		actualReader, err = hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 
@@ -1185,7 +1181,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	opts := ObjectOptions{}
@@ -1201,13 +1197,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		if hasServerSideEncryptionHeader(r.Header) && !hasSuffix(object, slashSeparator) { // handle SSE requests
 			reader, err = EncryptRequest(hashReader, r, bucket, object, metadata)
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 			info := ObjectInfo{Size: size}
 			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size) // do not try to verify encrypted content
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}
@@ -1223,7 +1219,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	// Create the object..
 	objInfo, err := putObject(ctx, bucket, object, hashReader, metadata, opts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -1255,20 +1251,15 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	// Notify object created event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedPut,
-		BucketName: bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedPut,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
-
-	for k, v := range objInfo.UserDefined {
-		logger.GetReqInfo(ctx).SetTags(k, v)
-	}
-
-	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 /// Multipart objectAPIHandlers
@@ -1282,7 +1273,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "NewMultipartUpload")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -1325,7 +1316,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	if objectAPI.IsEncryptionSupported() {
 		if hasServerSideEncryptionHeader(r.Header) {
 			if err := setEncryptionMetadata(r, bucket, object, encMetadata); err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 			// Set this for multipart only operations, we need to differentiate during
@@ -1361,7 +1352,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	}
 	uploadID, err := newMultipartUpload(ctx, bucket, object, metadata, opts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -1376,7 +1367,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObjectPart")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -1411,6 +1402,11 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	uploadID := r.URL.Query().Get("uploadId")
 	partIDString := r.URL.Query().Get("partNumber")
 
@@ -1458,7 +1454,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 	gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, readLock, srcOpts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	defer gr.Close()
@@ -1468,7 +1464,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	if crypto.IsEncrypted(srcInfo.UserDefined) {
 		actualPartSize, err = srcInfo.DecryptedSize()
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 	}
@@ -1485,8 +1481,11 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	}
 
 	// Get the object offset & length
-	startOffset, length, _ := rs.GetOffsetLength(actualPartSize)
-	actualPartSize = length
+	startOffset, length, err := rs.GetOffsetLength(actualPartSize)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		return
+	}
 
 	/// maximum copy size for multipart objects in a single operation
 	if isMaxAllowedPartSize(length) {
@@ -1494,15 +1493,16 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
+	actualPartSize = length
 	var reader io.Reader
-	var getLength = length
 
 	var li ListPartsInfo
 	li, err = objectAPI.ListObjectParts(ctx, dstBucket, dstObject, uploadID, 0, 1)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
+
 	// Read compression metadata preserved in the init multipart for the decision.
 	_, compressPart := li.UserDefined[ReservedMetadataPrefix+"compression"]
 	isCompressed := compressPart
@@ -1529,13 +1529,17 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualPartSize)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
 	if objectAPI.IsEncryptionSupported() && !isCompressed {
 		if crypto.IsEncrypted(li.UserDefined) {
-			if !hasServerSideEncryptionHeader(r.Header) {
+			if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(li.UserDefined) {
+				writeErrorResponse(w, ErrSSEMultipartEncrypted, r.URL)
+				return
+			}
+			if crypto.S3.IsEncrypted(li.UserDefined) && crypto.SSEC.IsRequested(r.Header) {
 				writeErrorResponse(w, ErrSSEMultipartEncrypted, r.URL)
 				return
 			}
@@ -1543,14 +1547,14 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			if crypto.SSEC.IsRequested(r.Header) {
 				key, err = ParseSSECustomerRequest(r)
 				if err != nil {
-					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 					return
 				}
 			}
 			var objectEncryptionKey []byte
 			objectEncryptionKey, err = decryptObjectInfo(key, dstBucket, dstObject, li.UserDefined)
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 
@@ -1562,25 +1566,25 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			partEncryptionKey := mac.Sum(nil)
 			reader, err = sio.EncryptReader(reader, sio.Config{Key: partEncryptionKey})
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 
 			info := ObjectInfo{Size: length}
-			size := info.EncryptedSize()
-			srcInfo.Reader, err = hash.NewReader(reader, size, "", "", actualPartSize)
+			srcInfo.Reader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", length)
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}
 	}
+
 	// Copy source object to destination, if source and destination
 	// object is same then only metadata is updated.
-	partInfo, err := objectAPI.CopyObjectPart(ctx, srcBucket, srcObject, dstBucket,
-		dstObject, uploadID, partID, startOffset, getLength, srcInfo, srcOpts, dstOpts)
+	partInfo, err := objectAPI.CopyObjectPart(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID,
+		startOffset, length, srcInfo, srcOpts, dstOpts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -1595,7 +1599,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectPart")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -1637,7 +1641,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			}
 			size, err = strconv.ParseInt(sizeStr[0], 10, 64)
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}
@@ -1712,7 +1716,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	var li ListPartsInfo
 	li, err = objectAPI.ListObjectParts(ctx, bucket, object, uploadID, 0, 1)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	// Read compression metadata preserved in the init multipart for the decision.
@@ -1726,7 +1730,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		var actualReader *hash.Reader
 		actualReader, err = hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 
@@ -1747,7 +1751,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 
@@ -1765,7 +1769,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		var li ListPartsInfo
 		li, err = objectAPI.ListObjectParts(ctx, bucket, object, uploadID, 0, 1)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			return
 		}
 		if crypto.IsEncrypted(li.UserDefined) {
@@ -1778,7 +1782,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			if crypto.SSEC.IsRequested(r.Header) {
 				key, err = ParseSSECustomerRequest(r)
 				if err != nil {
-					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 					return
 				}
 			}
@@ -1787,7 +1791,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			var objectEncryptionKey []byte
 			objectEncryptionKey, err = decryptObjectInfo(key, bucket, object, li.UserDefined)
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 
@@ -1800,14 +1804,14 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 			reader, err = sio.EncryptReader(hashReader, sio.Config{Key: partEncryptionKey})
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 
 			info := ObjectInfo{Size: size}
 			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size) // do not try to verify encrypted content
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 				return
 			}
 		}
@@ -1820,7 +1824,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, hashReader, opts)
 	if err != nil {
 		// Verify if the underlying error is signature mismatch.
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	if isCompressed {
@@ -1839,7 +1843,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AbortMultipartUpload")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1874,9 +1878,10 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 		return
 	}
 	if err := abortMultipartUpload(ctx, bucket, object, uploadID); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
+
 	writeSuccessNoContent(w)
 }
 
@@ -1884,7 +1889,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectParts")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1916,7 +1921,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 	}
 	listPartsInfo, err := objectAPI.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		return
 	}
 	response := generateListPartsResponse(listPartsInfo)
@@ -1930,7 +1935,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CompleteMultipartUpload")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -2005,7 +2010,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 			writePartSmallErrorResponse(w, r, oErr)
 		default:
 			// Handle all other generic issues.
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 		}
 		return
 	}
@@ -2034,20 +2039,15 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	// Notify object created event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedCompleteMultipartUpload,
-		BucketName: bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedCompleteMultipartUpload,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
-
-	for k, v := range objInfo.UserDefined {
-		logger.GetReqInfo(ctx).SetTags(k, v)
-	}
-
-	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 /// Delete objectAPIHandlers
@@ -2056,7 +2056,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteObject")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -2087,25 +2087,21 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			if err == dns.ErrNoEntriesFound {
 				writeErrorResponse(w, ErrNoSuchBucket, r.URL)
 			} else {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
 			}
 			return
 		}
-	} else {
-		getBucketInfo := objectAPI.GetBucketInfo
-		if api.CacheAPI() != nil {
-			getBucketInfo = api.CacheAPI().GetBucketInfo
-		}
-		if _, err := getBucketInfo(ctx, bucket); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-			return
-		}
 	}
 
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
-	// Ignore delete object errors while replying to client, since we are
-	// suppposed to reply only 204. Additionally log the error for
-	// investigation.
-	deleteObject(ctx, objectAPI, api.CacheAPI(), bucket, object, r)
+	if err := deleteObject(ctx, objectAPI, api.CacheAPI(), bucket, object, r); err != nil {
+		switch toAPIErrorCode(ctx, err) {
+		case ErrNoSuchBucket:
+			// When bucket doesn't exist specially handle it.
+			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+			return
+		}
+		// Ignore delete object errors while replying to client, since we are suppposed to reply only 204.
+	}
 	writeSuccessNoContent(w)
 }

cmd/object-handlers_test.go

@@ -2163,23 +2163,6 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		}
 	}
 
-	// Test for Anonymous/unsigned http request.
-	newCopyAnonObject := "new-anon-obj"
-	anonReq, err := newTestRequest("PUT", getCopyObjectURL("", bucketName, newCopyAnonObject), 0, nil)
-	if err != nil {
-		t.Fatalf("Minio %s: Failed to create an anonymous request for %s/%s: <ERROR> %v",
-			instanceType, bucketName, "new-anon-obj", err)
-	}
-
-	// Below is how CopyObjectHandler is registered.
-	// bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?")
-	// Its necessary to set the "X-Amz-Copy-Source" header for the request to be accepted by the handler.
-	anonReq.Header.Set("X-Amz-Copy-Source", url.QueryEscape("/"+bucketName+"/"+anonObject))
-	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
-	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
-	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPICopyObjectHandler", bucketName, newCopyAnonObject, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, newCopyAnonObject))
-
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
 	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
@@ -2550,6 +2533,8 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 	successResponse := generateCompleteMultpartUploadResponse(bucketName, objectName, getGetObjectURL("", bucketName, objectName), s3MD5)
 	encodedSuccessResponse := encodeResponse(successResponse)
 
+	ctx := context.Background()
+
 	testCases := []struct {
 		bucket    string
 		object    string
@@ -2572,7 +2557,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			accessKey: credentials.AccessKey,
 			secretKey: credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidPart{})),
+			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(ctx, InvalidPart{})),
 				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
@@ -2602,7 +2587,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			accessKey: credentials.AccessKey,
 			secretKey: credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidUploadID{UploadID: "abc"})),
+			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(ctx, InvalidUploadID{UploadID: "abc"})),
 				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
@@ -2617,7 +2602,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(completeMultipartAPIError{int64(4), int64(5242880), 1, "e2fc714c4727ee9395f324cd2e7f331f",
-				getAPIErrorResponse(getAPIError(toAPIErrorCode(PartTooSmall{PartNumber: 1})),
+				getAPIErrorResponse(getAPIError(toAPIErrorCode(ctx, PartTooSmall{PartNumber: 1})),
 					getGetObjectURL("", bucketName, objectName), "")}),
 			expectedRespStatus: http.StatusBadRequest,
 		},
@@ -2631,7 +2616,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			accessKey: credentials.AccessKey,
 			secretKey: credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidPart{})),
+			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(ctx, InvalidPart{})),
 				getGetObjectURL("", bucketName, objectName), "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},

cmd/peer-rpc-client-target.go

@@ -44,7 +44,7 @@ func (target *PeerRPCClientTarget) Close() error {
 // NewPeerRPCClientTarget - creates RPCClient target with given target ID available in remote peer.
 func NewPeerRPCClientTarget(bucketName string, targetID event.TargetID, rpcClient *PeerRPCClient) *PeerRPCClientTarget {
 	return &PeerRPCClientTarget{
-		id:             event.TargetID{targetID.ID, targetID.Name + "+" + mustGetUUID()},
+		id:             event.TargetID{ID: targetID.ID, Name: targetID.Name + "+" + mustGetUUID()},
 		remoteTargetID: targetID,
 		bucketName:     bucketName,
 		rpcClient:      rpcClient,

cmd/posix.go

@@ -847,7 +847,7 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 	return int64(len(buffer)), nil
 }
 
-func (s *posix) createFile(volume, path string) (f *os.File, err error) {
+func (s *posix) openFile(volume, path string, mode int) (f *os.File, err error) {
 	defer func() {
 		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
@@ -896,7 +896,7 @@ func (s *posix) createFile(volume, path string) (f *os.File, err error) {
 		}
 	}
 
-	w, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
+	w, err := os.OpenFile(filePath, mode, 0666)
 	if err != nil {
 		// File path cannot be verified since one of the parents is a file.
 		switch {
@@ -941,7 +941,7 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 	}
 
 	// Create file if not found
-	w, err := s.createFile(volume, path)
+	w, err := s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY)
 	if err != nil {
 		return err
 	}
@@ -972,6 +972,30 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 	return nil
 }
 
+func (s *posix) WriteAll(volume, path string, buf []byte) (err error) {
+	defer func() {
+		if err == errFaultyDisk {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
+		return errFaultyDisk
+	}
+
+	// Create file if not found
+	w, err := s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_WRONLY)
+	if err != nil {
+		return err
+	}
+
+	if _, err = w.Write(buf); err != nil {
+		return err
+	}
+
+	return w.Close()
+}
+
 // AppendFile - append a byte array at path, if file doesn't exist at
 // path this call explicitly creates it.
 func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
@@ -986,13 +1010,16 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 	}
 
 	// Create file if not found
-	w, err := s.createFile(volume, path)
+	w, err := s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY)
 	if err != nil {
 		return err
 	}
-	_, err = w.Write(buf)
-	w.Close()
-	return err
+
+	if _, err = w.Write(buf); err != nil {
+		return err
+	}
+
+	return w.Close()
 }
 
 // StatFile - get file info.

cmd/server-main.go

@@ -221,17 +221,21 @@ func serverMain(ctx *cli.Context) {
 	// Handle all server command args.
 	serverHandleCmdArgs(ctx)
 
-	// Handle all server environment vars.
-	serverHandleEnvVars()
-
 	// Create certs path.
 	logger.FatalIf(createConfigDir(), "Unable to initialize configuration files")
 
-	// Check and load SSL certificates.
+	// Check and load TLS certificates.
 	var err error
-	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
+	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
 	logger.FatalIf(err, "Unable to load the TLS configuration")
 
+	// Check and load Root CAs.
+	globalRootCAs, err = getRootCAs(getCADir())
+	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
+
+	// Handle all server environment vars.
+	serverHandleEnvVars()
+
 	// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
 	if globalIsDistXL {
 		if globalEndpoints.IsHTTPS() && !globalIsSSL {

cmd/signals.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"os"
+	"strings"
 
 	"github.com/minio/minio/cmd/logger"
 )
@@ -71,7 +72,7 @@ func handleSignals() {
 			exit(err == nil && oerr == nil)
 		case osSignal := <-globalOSSignalCh:
 			stopHTTPTrace()
-			logger.Info("Exiting on signal %v", osSignal)
+			logger.Info("Exiting on signal: %s", strings.ToUpper(osSignal.String()))
 			exit(stopProcess())
 		case signal := <-globalServiceSignalCh:
 			switch signal {

cmd/signature-v2.go

@@ -69,15 +69,9 @@ var resourceList = []string{
 func doesPolicySignatureV2Match(formValues http.Header) APIErrorCode {
 	cred := globalServerConfig.GetCredential()
 	accessKey := formValues.Get("AWSAccessKeyId")
-	if accessKey != cred.AccessKey {
-		if globalIAMSys == nil {
-			return ErrInvalidAccessKeyID
-		}
-		var ok bool
-		cred, ok = globalIAMSys.GetUser(accessKey)
-		if !ok {
-			return ErrInvalidAccessKeyID
-		}
+	cred, _, s3Err := checkKeyValid(accessKey)
+	if s3Err != ErrNone {
+		return s3Err
 	}
 	policy := formValues.Get("Policy")
 	signature := formValues.Get("Signature")
@@ -153,16 +147,9 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 		return ErrInvalidQueryParams
 	}
 
-	// Validate if access key id same.
-	if accessKey != cred.AccessKey {
-		if globalIAMSys == nil {
-			return ErrInvalidAccessKeyID
-		}
-		var ok bool
-		cred, ok = globalIAMSys.GetUser(accessKey)
-		if !ok {
-			return ErrInvalidAccessKeyID
-		}
+	cred, _, s3Err := checkKeyValid(accessKey)
+	if s3Err != ErrNone {
+		return s3Err
 	}
 
 	// Make sure the request has not expired.
@@ -189,27 +176,25 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 	return ErrNone
 }
 
-func getReqAccessKeyV2(r *http.Request) (string, bool, APIErrorCode) {
+func getReqAccessKeyV2(r *http.Request) (auth.Credentials, bool, APIErrorCode) {
 	if accessKey := r.URL.Query().Get("AWSAccessKeyId"); accessKey != "" {
-		owner, s3Err := checkKeyValid(accessKey)
-		return accessKey, owner, s3Err
+		return checkKeyValid(accessKey)
 	}
 
 	// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
 	// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
 	authFields := strings.Split(r.Header.Get("Authorization"), " ")
 	if len(authFields) != 2 {
-		return "", false, ErrMissingFields
+		return auth.Credentials{}, false, ErrMissingFields
 	}
 
 	// Then will be splitting on ":", this will seprate `AWSAccessKeyId` and `Signature` string.
 	keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":")
 	if len(keySignFields) != 2 {
-		return "", false, ErrMissingFields
+		return auth.Credentials{}, false, ErrMissingFields
 	}
 
-	owner, s3Err := checkKeyValid(keySignFields[0])
-	return keySignFields[0], owner, s3Err
+	return checkKeyValid(keySignFields[0])
 }
 
 // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
@@ -244,24 +229,11 @@ func validateV2AuthHeader(r *http.Request) (auth.Credentials, APIErrorCode) {
 		return cred, ErrSignatureVersionNotSupported
 	}
 
-	accessKey, owner, apiErr := getReqAccessKeyV2(r)
+	cred, _, apiErr := getReqAccessKeyV2(r)
 	if apiErr != ErrNone {
 		return cred, apiErr
 	}
 
-	cred = globalServerConfig.GetCredential()
-	// Access credentials.
-	if !owner {
-		if globalIAMSys == nil {
-			return cred, ErrInvalidAccessKeyID
-		}
-		var ok bool
-		cred, ok = globalIAMSys.GetUser(accessKey)
-		if !ok {
-			return cred, ErrInvalidAccessKeyID
-		}
-	}
-
 	return cred, ErrNone
 }
 

cmd/signature-v4-parser.go

@@ -47,22 +47,21 @@ func (c credentialHeader) getScope() string {
 	}, "/")
 }
 
-func getReqAccessKeyV4(r *http.Request, region string) (string, bool, APIErrorCode) {
+func getReqAccessKeyV4(r *http.Request, region string) (auth.Credentials, bool, APIErrorCode) {
 	ch, err := parseCredentialHeader("Credential="+r.URL.Query().Get("X-Amz-Credential"), region)
 	if err != ErrNone {
 		// Strip off the Algorithm prefix.
 		v4Auth := strings.TrimPrefix(r.Header.Get("Authorization"), signV4Algorithm)
 		authFields := strings.Split(strings.TrimSpace(v4Auth), ",")
 		if len(authFields) != 3 {
-			return "", false, ErrMissingFields
+			return auth.Credentials{}, false, ErrMissingFields
 		}
 		ch, err = parseCredentialHeader(authFields[0], region)
 		if err != ErrNone {
-			return "", false, err
+			return auth.Credentials{}, false, err
 		}
 	}
-	owner, s3Err := checkKeyValid(ch.accessKey)
-	return ch.accessKey, owner, s3Err
+	return checkKeyValid(ch.accessKey)
 }
 
 // parse credentialHeader string into its structured form.

cmd/signature-v4-utils.go

@@ -22,6 +22,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/sha256-simd"
 )
 
@@ -102,19 +103,21 @@ func isValidRegion(reqRegion string, confRegion string) bool {
 
 // check if the access key is valid and recognized, additionally
 // also returns if the access key is owner/admin.
-func checkKeyValid(accessKey string) (bool, APIErrorCode) {
+func checkKeyValid(accessKey string) (auth.Credentials, bool, APIErrorCode) {
 	var owner = true
-	if globalServerConfig.GetCredential().AccessKey != accessKey {
+	var cred = globalServerConfig.GetCredential()
+	if cred.AccessKey != accessKey {
 		if globalIAMSys == nil {
-			return false, ErrInvalidAccessKeyID
+			return cred, false, ErrInvalidAccessKeyID
 		}
 		// Check if the access key is part of users credentials.
-		if _, ok := globalIAMSys.GetUser(accessKey); !ok {
-			return false, ErrInvalidAccessKeyID
+		var ok bool
+		if cred, ok = globalIAMSys.GetUser(accessKey); !ok {
+			return cred, false, ErrInvalidAccessKeyID
 		}
 		owner = false
 	}
-	return owner, ErrNone
+	return cred, owner, ErrNone
 }
 
 // sumHMAC calculate hmac between two input byte array.

cmd/signature-v4.go

@@ -161,9 +161,6 @@ func compareSignatureV4(sig1, sig2 string) bool {
 //     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
 // returns ErrNone if the signature matches.
 func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
-	// Access credentials.
-	cred := globalServerConfig.GetCredential()
-
 	// Server region.
 	region := globalServerConfig.GetRegion()
 
@@ -173,16 +170,9 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
 		return ErrMissingFields
 	}
 
-	// Verify if the access key id matches.
-	if credHeader.accessKey != cred.AccessKey {
-		if globalIAMSys == nil {
-			return ErrInvalidAccessKeyID
-		}
-		var ok bool
-		cred, ok = globalIAMSys.GetUser(credHeader.accessKey)
-		if !ok {
-			return ErrInvalidAccessKeyID
-		}
+	cred, _, s3Err := checkKeyValid(credHeader.accessKey)
+	if s3Err != ErrNone {
+		return s3Err
 	}
 
 	// Get signing key.
@@ -204,9 +194,6 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
 //     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
 // returns ErrNone if the signature matches.
 func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region string) APIErrorCode {
-	// Access credentials.
-	cred := globalServerConfig.GetCredential()
-
 	// Copy request
 	req := *r
 
@@ -216,16 +203,9 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 		return err
 	}
 
-	// Verify if the access key id matches.
-	if pSignValues.Credential.accessKey != cred.AccessKey {
-		if globalIAMSys == nil {
-			return ErrInvalidAccessKeyID
-		}
-		var ok bool
-		cred, ok = globalIAMSys.GetUser(pSignValues.Credential.accessKey)
-		if !ok {
-			return ErrInvalidAccessKeyID
-		}
+	cred, _, s3Err := checkKeyValid(pSignValues.Credential.accessKey)
+	if s3Err != ErrNone {
+		return s3Err
 	}
 
 	// Extract all the signed headers along with its values.
@@ -233,6 +213,7 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	if errCode != ErrNone {
 		return errCode
 	}
+
 	// Construct new query.
 	query := make(url.Values)
 	if req.URL.Query().Get("X-Amz-Content-Sha256") != "" {
@@ -326,9 +307,6 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 //     - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
 // returns ErrNone if signature matches.
 func doesSignatureMatch(hashedPayload string, r *http.Request, region string) APIErrorCode {
-	// Access credentials.
-	cred := globalServerConfig.GetCredential()
-
 	// Copy request.
 	req := *r
 
@@ -347,16 +325,9 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string) AP
 		return errCode
 	}
 
-	// Verify if the access key id matches.
-	if signV4Values.Credential.accessKey != cred.AccessKey {
-		if globalIAMSys == nil {
-			return ErrInvalidAccessKeyID
-		}
-		var ok bool
-		cred, ok = globalIAMSys.GetUser(signV4Values.Credential.accessKey)
-		if !ok {
-			return ErrInvalidAccessKeyID
-		}
+	cred, _, s3Err := checkKeyValid(signV4Values.Credential.accessKey)
+	if s3Err != ErrNone {
+		return s3Err
 	}
 
 	// Extract date, if not present throw error.
@@ -366,6 +337,7 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string) AP
 			return ErrMissingDateHeader
 		}
 	}
+
 	// Parse date header.
 	t, e := time.Parse(iso8601Format, date)
 	if e != nil {

cmd/storage-interface.go

@@ -47,6 +47,9 @@ type StorageAPI interface {
 	StatFile(volume string, path string) (file FileInfo, err error)
 	DeleteFile(volume string, path string) (err error)
 
+	// Write all data, syncs the data to disk.
+	WriteAll(volume string, path string, buf []byte) (err error)
+
 	// Read all.
 	ReadAll(volume string, path string) (buf []byte, err error)
 }

cmd/storage-rest-client.go

@@ -221,6 +221,17 @@ func (client *storageRESTClient) AppendFile(volume, path string, buffer []byte)
 	return err
 }
 
+// WriteAll - write all data to a file.
+func (client *storageRESTClient) WriteAll(volume, path string, buffer []byte) error {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	reader := bytes.NewBuffer(buffer)
+	respBody, err := client.call(storageRESTMethodWriteAll, values, reader)
+	defer CloseResponse(respBody)
+	return err
+}
+
 // StatFile - stat a file.
 func (client *storageRESTClient) StatFile(volume, path string) (info FileInfo, err error) {
 	values := make(url.Values)

cmd/storage-rest-common.go

@@ -28,6 +28,7 @@ const (
 
 	storageRESTMethodPrepareFile = "preparefile"
 	storageRESTMethodAppendFile  = "appendfile"
+	storageRESTMethodWriteAll    = "writeall"
 	storageRESTMethodStatFile    = "statfile"
 	storageRESTMethodReadAll     = "readall"
 	storageRESTMethodReadFile    = "readfile"

cmd/storage-rest-server.go

@@ -179,6 +179,33 @@ func (s *storageRESTServer) AppendFileHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
+// WriteAllHandler - write to file all content.
+func (s *storageRESTServer) WriteAllHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	filePath := vars[storageRESTFilePath]
+
+	if r.ContentLength < 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	buf := make([]byte, r.ContentLength)
+	_, err := io.ReadFull(r.Body, buf)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	err = s.storage.WriteAll(volume, filePath, buf)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+	}
+}
+
 // StatFileHandler - stat a file.
 func (s *storageRESTServer) StatFileHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -336,6 +363,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
 			Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTLength)...)
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodAppendFile).HandlerFunc(httpTraceHdrs(server.AppendFileHandler)).
 			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodWriteAll).HandlerFunc(httpTraceHdrs(server.WriteAllHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodStatFile).HandlerFunc(httpTraceHdrs(server.StatFileHandler)).
 			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodReadAll).HandlerFunc(httpTraceHdrs(server.ReadAllHandler)).

cmd/streaming-signature-v4.go

@@ -91,17 +91,9 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 		return cred, "", "", time.Time{}, errCode
 	}
 
-	cred = globalServerConfig.GetCredential()
-	// Verify if the access key id matches.
-	if signV4Values.Credential.accessKey != cred.AccessKey {
-		if globalIAMSys == nil {
-			return cred, "", "", time.Time{}, ErrInvalidAccessKeyID
-		}
-		var ok bool
-		cred, ok = globalIAMSys.GetUser(signV4Values.Credential.accessKey)
-		if !ok {
-			return cred, "", "", time.Time{}, ErrInvalidAccessKeyID
-		}
+	cred, _, errCode = checkKeyValid(signV4Values.Credential.accessKey)
+	if errCode != ErrNone {
+		return cred, "", "", time.Time{}, errCode
 	}
 
 	// Verify if region is valid.
@@ -114,6 +106,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 			return cred, "", "", time.Time{}, ErrMissingDateHeader
 		}
 	}
+
 	// Parse date header.
 	var err error
 	date, err = time.Parse(iso8601Format, dateStr)

cmd/sts-handlers.go

@@ -162,8 +162,17 @@ func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *
 		return
 	}
 
+	// JWT has requested a custom claim with policy value set.
+	// This is a Minio STS API specific value, this value should
+	// be set and configured on your identity provider as part of
+	// JWT custom claims.
+	var policyName string
+	if v, ok := m["policy"]; ok {
+		policyName, _ = v.(string)
+	}
+
 	// Set the newly generated credentials.
-	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred); err != nil {
+	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
 		logger.LogIf(ctx, err)
 		writeSTSErrorResponse(w, ErrSTSInternalError)
 		return

cmd/test-utils_test.go

@@ -2354,8 +2354,9 @@ func TestToErrIsNil(t *testing.T) {
 	if toStorageErr(nil) != nil {
 		t.Errorf("Test expected to return nil, failed instead got a non-nil value %s", toStorageErr(nil))
 	}
-	if toAPIErrorCode(nil) != ErrNone {
-		t.Errorf("Test expected error code to be ErrNone, failed instead provided %d", toAPIErrorCode(nil))
+	ctx := context.Background()
+	if toAPIErrorCode(ctx, nil) != ErrNone {
+		t.Errorf("Test expected error code to be ErrNone, failed instead provided %d", toAPIErrorCode(ctx, nil))
 	}
 }
 

cmd/update-main.go

@@ -253,12 +253,6 @@ func getUserAgent(mode string) string {
 	if mode != "" {
 		uaAppend("; ", mode)
 	}
-	if len(globalCacheDrives) > 0 {
-		uaAppend("; ", "feature-cache")
-	}
-	if globalWORMEnabled {
-		uaAppend("; ", "feature-worm")
-	}
 	if IsDCOS() {
 		uaAppend("; ", "dcos")
 	}

cmd/web-handlers.go

@@ -707,7 +707,7 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs,
 func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "WebUpload")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
@@ -830,27 +830,22 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 
 	// Notify object created event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedPut,
-		BucketName: bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedPut,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
-
-	for k, v := range objInfo.UserDefined {
-		logger.GetReqInfo(ctx).SetTags(k, v)
-	}
-
-	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // Download - file download handler.
 func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "WebDownload")
 
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()
@@ -1012,12 +1007,6 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 		Host:         host,
 		Port:         port,
 	})
-
-	for k, v := range objInfo.UserDefined {
-		logger.GetReqInfo(ctx).SetTags(k, v)
-	}
-
-	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // DownloadZipArgs - Argument for downloading a bunch of files as a zip file.
@@ -1038,8 +1027,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 	}
 
 	ctx := newContext(r, w, "WebDownloadZip")
-
-	defer logger.AuditLog(ctx, r)
+	defer logger.AuditLog(ctx, w, r)
 
 	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()

cmd/xl-sets.go

@@ -1202,8 +1202,14 @@ func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun bool) (re
 	for _, endpoint := range s.endpoints {
 		var foundBefore bool
 		for _, v := range res.Before.Drives {
-			if v.Endpoint == endpoint.String() {
-				foundBefore = true
+			if endpoint.IsLocal {
+				if v.Endpoint == endpoint.Path {
+					foundBefore = true
+				}
+			} else {
+				if v.Endpoint == endpoint.String() {
+					foundBefore = true
+				}
 			}
 		}
 		if !foundBefore {
@@ -1215,8 +1221,14 @@ func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun bool) (re
 		}
 		var foundAfter bool
 		for _, v := range res.After.Drives {
-			if v.Endpoint == endpoint.String() {
-				foundAfter = true
+			if endpoint.IsLocal {
+				if v.Endpoint == endpoint.Path {
+					foundAfter = true
+				}
+			} else {
+				if v.Endpoint == endpoint.String() {
+					foundAfter = true
+				}
 			}
 		}
 		if !foundAfter {

cmd/xl-v1-bucket.go

@@ -244,7 +244,6 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
 			err := disk.DeleteVol(bucket)
 
 			if err != nil {
-				logger.LogIf(ctx, err)
 				dErrs[index] = err
 				return
 			}

cmd/xl-v1-healing.go

@@ -36,29 +36,6 @@ func (xl xlObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRes
 	return madmin.HealResultItem{}, NotImplemented{}
 }
 
-// checks for bucket if it exists in writeQuorum number of disks, this call
-// is only used by healBucket().
-func checkBucketExistsInQuorum(ctx context.Context, storageDisks []StorageAPI, bucketName string) (err error) {
-	var wg = &sync.WaitGroup{}
-
-	errs := make([]error, len(storageDisks))
-	// Prepare object creation in a all disks
-	for index, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		wg.Add(1)
-		go func(index int, disk StorageAPI) {
-			defer wg.Done()
-			_, errs[index] = disk.StatVol(bucketName)
-		}(index, disk)
-	}
-	wg.Wait()
-
-	readQuorum := len(storageDisks) / 2
-	return reduceWriteQuorumErrs(ctx, errs, nil, readQuorum)
-}
-
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
@@ -67,13 +44,6 @@ func (xl xlObjects) HealBucket(ctx context.Context, bucket string, dryRun bool)
 
 	storageDisks := xl.getDisks()
 
-	// Check if bucket doesn't exist in writeQuorum number of disks, if quorum
-	// number of disks returned that bucket does not exist we quickly return
-	// and do not proceed to heal.
-	if err = checkBucketExistsInQuorum(ctx, storageDisks, bucket); err != nil {
-		return results, err
-	}
-
 	// get write quorum for an object
 	writeQuorum := len(storageDisks)/2 + 1
 

cmd/xl-v1-metadata.go

@@ -423,8 +423,9 @@ func writeXLMetadata(ctx context.Context, disk StorageAPI, bucket, prefix string
 		logger.LogIf(ctx, err)
 		return err
 	}
+
 	// Persist marshaled data.
-	err = disk.AppendFile(bucket, jsonFile, metadataBytes)
+	err = disk.WriteAll(bucket, jsonFile, metadataBytes)
 	logger.LogIf(ctx, err)
 	return err
 }

cmd/xl-v1-multipart.go

@@ -385,7 +385,6 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	// Should return IncompleteBody{} error when reader has fewer bytes
 	// than specified in request header.
 	if n < data.Size() {
-		logger.LogIf(ctx, IncompleteBody{})
 		return pi, IncompleteBody{}
 	}
 
@@ -655,7 +654,6 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 				PartNumber: part.PartNumber,
 				GotETag:    part.ETag,
 			}
-			logger.LogIf(ctx, invp)
 			return oi, invp
 		}
 
@@ -666,17 +664,11 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 				ExpETag:    currentXLMeta.Parts[partIdx].ETag,
 				GotETag:    part.ETag,
 			}
-			logger.LogIf(ctx, invp)
 			return oi, invp
 		}
 
 		// All parts except the last part has to be atleast 5MB.
 		if (i < len(parts)-1) && !isMinAllowedPartSize(currentXLMeta.Parts[partIdx].ActualSize) {
-			logger.LogIf(ctx, PartTooSmall{
-				PartNumber: part.PartNumber,
-				PartSize:   currentXLMeta.Parts[partIdx].ActualSize,
-				PartETag:   part.ETag,
-			})
 			return oi, PartTooSmall{
 				PartNumber: part.PartNumber,
 				PartSize:   currentXLMeta.Parts[partIdx].ActualSize,

cmd/xl-v1-object.go

@@ -984,13 +984,13 @@ func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (er
 		return err
 	}
 
-	if !xl.isObject(bucket, object) && !xl.isObjectDir(bucket, object) {
-		return ObjectNotFound{bucket, object}
-	}
-
 	var writeQuorum int
 	var isObjectDir = hasSuffix(object, slashSeparator)
 
+	if isObjectDir && !xl.isObjectDir(bucket, object) {
+		return toObjectErr(errFileNotFound, bucket, object)
+	}
+
 	if isObjectDir {
 		writeQuorum = len(xl.getDisks())/2 + 1
 	} else {

cmd/xl-v1-utils.go

@@ -305,12 +305,15 @@ func readXLMeta(ctx context.Context, disk StorageAPI, bucket string, object stri
 	// Reads entire `xl.json`.
 	xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile))
 	if err != nil {
-		if err != errFileNotFound {
+		if err != errFileNotFound && err != errVolumeNotFound {
 			logger.GetReqInfo(ctx).AppendTags("disk", disk.String())
 			logger.LogIf(ctx, err)
 		}
 		return xlMetaV1{}, err
 	}
+	if len(xlMetaBuf) == 0 {
+		return xlMetaV1{}, errFileNotFound
+	}
 	// obtain xlMetaV1{} using `github.com/tidwall/gjson`.
 	xlMeta, err = xlMetaV1UnmarshalJSON(ctx, xlMetaBuf)
 	if err != nil {

docs/bucket/notifications/README.md

@@ -17,6 +17,7 @@ Bucket events can be published to the following targets:
 | [`AMQP`](#AMQP) | [`Redis`](#Redis) | [`MySQL`](#MySQL) |
 | [`MQTT`](#MQTT) | [`NATS`](#NATS) | [`Apache Kafka`](#apache-kafka) |
 | [`Elasticsearch`](#Elasticsearch) | [`PostgreSQL`](#PostgreSQL) | [`Webhooks`](#webhooks) |
+| [`NSQ`](#NSQ) | | |
 
 ## Prerequisites
 
@@ -1007,5 +1008,77 @@ mc ls myminio/images-thumbnail
 [2017-02-08 11:39:40 IST]   992B images-thumbnail.jpg
 ```
 
+<a name="NSQ"></a>
+## Publish Minio events to NSQ
+
+Install an NSQ Daemon from [here](https://nsq.io/). Or use the following Docker
+command for starting an nsq daemon:
+
+```
+docker run --rm -p 4150-4151:4150-4151 nsqio/nsq /nsqd
+```
+
+### Step 1: Add NSQ endpoint to Minio
+
+The Minio server configuration file is stored on the backend in json format. The NSQ configuration is located in the `nsq` key under the `notify` top-level key. Create a configuration key-value pair here for your NSQ instance. The key is a name for your NSQ endpoint, and the value is a collection of key-value parameters.
+
+An example configuration for NSQ is shown below:
+
+```json
+"nsq": {
+    "1": {
+        "enable": true,
+        "nsqdAddress": "127.0.0.1:4150",
+        "topic": "minio",
+        "tls": {
+            "enable": false,       
+            "skipVerify": true
+        }
+    }
+}
+```
+To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
+```sh
+$ mc admin config get myminio/ > /tmp/myconfig
+```
+After updating the NSQ configuration in /tmp/myconfig , use `mc admin config set` command to update the configuration for the deployment.Restart the Minio server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:nsq` at start-up if there were no errors.
+```sh
+$ mc admin config set myminio < /tmp/myconfig
+```
+
+Note that, you can add as many NSQ daemon endpoint configurations as needed by providing an identifier (like "1" in the example above) for the NSQ instance and an object of per-server configuration parameters.
+
+
+### Step 2: Enable bucket notification using Minio client
+
+We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted ``images`` bucket on ``myminio`` server. Here ARN value is ``arn:minio:sqs::1:nsq``.
+
+```
+mc mb myminio/images
+mc events add  myminio/images arn:minio:sqs::1:nsq --suffix .jpg
+mc events list myminio/images
+arn:minio:sqs::1:nsq s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
+```
+
+### Step 3: Test on NSQ
+
+The simplest test is to download `nsq_tail` from [nsq github](https://github.com/nsqio/nsq/releases)
+
+```
+./nsq_tail -nsqd-tcp-address 127.0.0.1:4150 -topic minio
+```
+
+Open another terminal and upload a JPEG image into ``images`` bucket.
+
+```
+mc cp gopher.jpg myminio/images
+```
+
+You should receive the following event notification via NSQ once the upload completes.
+
+```
+{"EventName":"s3:ObjectCreated:Put","Key":"images/gopher.jpg","Records":[{"eventVersion":"2.0","eventSource":"minio:s3","awsRegion":"","eventTime":"2018-10-31T09:31:11Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"21EJ9HYV110O8NVX2VMS"},"requestParameters":{"sourceIPAddress":"10.1.1.1"},"responseElements":{"x-amz-request-id":"1562A792DAA53426","x-minio-origin-endpoint":"http://10.0.3.1:9000"},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"21EJ9HYV110O8NVX2VMS"},"arn":"arn:aws:s3:::images"},"object":{"key":"gopher.jpg","size":162023,"eTag":"5337769ffa594e742408ad3f30713cd7","contentType":"image/jpeg","userMetadata":{"content-type":"image/jpeg"},"versionId":"1","sequencer":"1562A792DAA53426"}},"source":{"host":"","port":"","userAgent":"Minio (linux; amd64) minio-go/v6.0.8 mc/DEVELOPMENT.GOGET"}}]}
+```
+
 
 *NOTE* If you are running [distributed Minio](https://docs.minio.io/docs/distributed-minio-quickstart-guide), modify ``~/.minio/config.json`` on all the nodes with your bucket event notification backend configuration.

docs/config/config.sample.json

@@ -121,6 +121,17 @@
 				}
 			}
 		},
+		"nsq": {
+			"1": {
+				"enable": false,
+				"nsqdAddress": "",
+				"topic": "",
+				"tls": {
+					"enable": false,
+					"skipVerify": true
+				}
+			}
+		},
 		"postgresql": {
 			"1": {
 				"enable": false,

docs/kms/README.md

@@ -1,6 +1,6 @@
 # KMS Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
 
-KMS feature allows you to use Vault to generate and manages keys which are used by the minio server to encrypt objects.This document explains how to configure Minio with Vault as KMS.
+KMS feature allows you to use Vault to generate and manage encryption keys for use by the minio server to encrypt objects. This document explains how to configure Minio with Vault as KMS.
 
 ## Get started
 
@@ -10,13 +10,49 @@ Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quicks
 ### 2. Configure Vault
 Vault as Key Management System requires following to be configured in Vault
 
-- transit backend configured with a named encryption key-ring
-- AppRole based authentication with read/update policy for transit backend. In particular, read and update policy
-  are required for the generate data key endpoint and decrypt key endpoint.
+- [transit](https://www.vaultproject.io/docs/secrets/transit/index.html) backend configured with a named encryption key-ring
+- [AppRole](https://www.vaultproject.io/docs/auth/approle.html) based authentication with read/update policy for transit backend. In particular, read and update policy are required for the [Generate Data Key](https://www.vaultproject.io/api/secret/transit/index.html#generate-data-key) endpoint and [Decrypt Data](https://www.vaultproject.io/api/secret/transit/index.html#decrypt-data) endpoint.
+
+Here is a sample quick start for configuring vault with a transit backend and Approle with correct policy 
+#### 2.1 Start Vault server in dev mode
+In dev mode, Vault server runs in-memory and starts unsealed. Note that running Vault in dev mode is insecure and any data stored in the Vault is lost upon restart.
+```
+vault server -dev
+```
+
+#### 2.2 Set up vault transit backend and create an app role
+```
+cat > vaultpolicy.hcl <<EOF
+path "transit/datakey/plaintext/my-minio-key" { 
+  capabilities = [ "read", "update"]
+}
+path "transit/decrypt/my-minio-key" { 
+  capabilities = [ "read", "update"]
+}
+path "transit/encrypt/my-minio-key" { 
+  capabilities = [ "read", "update"]
+}
+
+EOF
+
+export VAULT_ADDR='http://127.0.0.1:8200'
+vault auth enable approle    # enable approle style auth
+vault secrets enable transit  # enable transit secrets engine
+vault write -f  transit/keys/my-minio-key  #define a encryption key-ring for the transit path
+vault policy write minio-policy ./vaultpolicy.hcl  #define a policy for AppRole to access transit path
+vault write auth/approle/role/my-role token_num_uses=0  secret_id_num_uses=0  period=60s # period indicates it is renewable if token is renewed before the period is over
+# define an AppRole
+vault write auth/approle/role/my-role policies=minio-policy # apply policy to role
+vault read auth/approle/role/my-role/role-id  # get Approle ID
+vault write -f auth/approle/role/my-role/secret-id
+
+```
+
+The AppRole ID, AppRole Secret Id, Vault endpoint and Vault key name can now be used to start minio server with Vault as KMS.
 
 ### 3. Environment variables
 
-You'll need the Vault endpoint, AppRole ID, AppRole SecretID, encryption key-ring name before starting Minio server with Vault as KMS
+You'll need the Vault endpoint, AppRole ID, AppRole SecretID and encryption key-ring name defined in step 2.2
 
 ```sh
 export MINIO_SSE_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
@@ -26,14 +62,14 @@ export MINIO_SSE_VAULT_KEY_NAME=my-minio-key
 minio server ~/export
 ```
 
-Optionally set `MINIO_SSE_VAULT_CAPATH` is the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.
+Optionally set `MINIO_SSE_VAULT_CAPATH` as the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.
 ```
 export MINIO_SSE_VAULT_CAPATH=/home/user/custom-pems
 ```
 
 ### 4. Test your setup
 
-To test this setup, access the Minio server via browser or [`mc`](https://docs.minio.io/docs/minio-client-quickstart-guide). You’ll see the uploaded files are accessible from the all the Minio endpoints.
+To test this setup, start minio server with environment variables set in Step 3, and server is ready to handle SSE-S3 requests.
 
 # Explore Further
 

docs/logging/README.md

@@ -0,0 +1,108 @@
+# Minio Logging Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+This document explains how to configure Minio server to log to different logging targets.
+
+## Log Targets
+Minio supports currently two target types
+
+- console
+- http
+
+### Console Target
+Console target logs to `/dev/stderr` and is enabled by default. To turn-off console logging you would have to update your Minio server configuration using `mc admin config set` command.
+
+Assuming `mc` is already [configured](https://docs.minio.io/docs/minio-client-quickstart-guide.html)
+```
+mc admin config get myminio/ > /tmp/config
+```
+
+Edit the `/tmp/config` and toggle `console` field `enabled` from `true` to `false`.
+
+```json
+	"logger": {
+		"console": {
+			"enabled": false
+		}
+	},
+```
+
+Once changed, now you may set the changed config to server through following commands.
+```
+mc admin config set myminio/ < /tmp/config
+mc admin restart myminio/
+```
+
+### HTTP Target
+HTTP target logs to a generic HTTP endpoint in JSON format and is not enabled by default. To enable HTTP target logging you would have to update your Minio server configuration using `mc admin config set` command.
+
+Assuming `mc` is already [configured](https://docs.minio.io/docs/minio-client-quickstart-guide.html)
+```
+mc admin config get myminio/ > /tmp/config
+```
+
+Edit the `/tmp/config` and toggle `http` field `enabled` from `false` to `true`.
+```json
+	"logger": {
+		"console": {
+			"enabled": false
+		},
+		"http": {
+			"1": {
+				"enabled": true,
+			    "endpoint": "http://endpoint:port/path"
+			}
+		}
+	},
+```
+NOTE: `http://endpoint:port/path` is a placeholder value to indicate the URL format, please change this accordingly as per your configuration.
+
+Once changed, now you may set the changed config to server through following commands.
+```
+mc admin config set myminio/ < /tmp/config
+mc admin restart myminio/
+```
+
+Minio also honors environment variable for HTTP target logging as shown below, this setting will override the endpoint settings in the Minio server config.
+```
+MINIO_LOGGER_HTTP_ENDPOINT=http://localhost:8080/minio/logs minio server /mnt/data
+```
+
+## Audit Targets
+For audit logging Minio supports only HTTP target type for now. Audit logging is currently only available through environment variable.
+```
+MINIO_AUDIT_LOGGER_HTTP_ENDPOINT=http://localhost:8080/minio/logs/audit minio server /mnt/data
+```
+
+Setting this environment variable automatically enables audit logging to the HTTP target. The audit logging is in JSON format as described below.
+```json
+{
+  "version": "1",
+  "deploymentid": "1b3002bf-5005-4d9b-853e-64a05008ebb2",
+  "time": "2018-11-02T21:57:58.231480177Z",
+  "api": {
+    "name": "ListBuckets",
+    "args": {}
+  },
+  "remotehost": "127.0.0.1",
+  "requestID": "15636D7C53428FD4",
+  "userAgent": "Minio (linux; amd64) minio-go/v6.0.8 mc/2018-11-02T21:13:30Z",
+  "requestHeader": {
+    "Authorization": "AWS4-HMAC-SHA256 Credential=minio/20181102/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6db486b42a85b23bffba66d654ce60242a7e92fb27cd4a1756e68082c02cc204",
+    "User-Agent": "Minio (linux; amd64) minio-go/v6.0.8 mc/2018-11-02T21:13:30Z",
+    "X-Amz-Content-Sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "X-Amz-Date": "20181102T215758Z"
+  },
+  "responseHeader": {
+    "Accept-Ranges": "bytes",
+    "Content-Security-Policy": "block-all-mixed-content",
+    "Content-Type": "application/xml",
+    "Server": "Minio/DEVELOPMENT.2018-11-02T21-57-15Z (linux; amd64)",
+    "Vary": "Origin",
+    "X-Amz-Request-Id": "15636D7C53428FD4",
+    "X-Xss-Protection": "1; mode=block"
+  }
+}
+```
+
+## Explore Further
+* [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide)
+* [Configure Minio Server with TLS](https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls)

docs/metrics/healthcheck/README.md

@@ -20,4 +20,4 @@ Platforms like Kubernetes *do not* forward traffic to a pod until its readiness
 
 ### Configuration example
 
-Sample `liveness` and `readiness` probe configuration in a Kubernetes `yaml` file can be found [here](https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-standalone-deployment.yaml).
+Sample `liveness` and `readiness` probe configuration in a Kubernetes `yaml` file can be found [here](https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes/minio-standalone-deployment.yaml).

docs/minio-limits.md

@@ -53,4 +53,4 @@ We found the following APIs to be redundant or less useful outside of AWS S3. If
 - ObjectVersions
 
 ### Object name restrictions on Minio
-Object names that contain characters `^*|\/"` are unsupported on Windows and other file systems which do not support filenames with these characters. Note that this list is not exhaustive, and depends on the maintainers of the filesystem itself.
+Object names that contain characters `^*|\/&";` are unsupported on Windows and other file systems which do not support filenames with these characters. Note that this list is not exhaustive, and depends on the maintainers of the filesystem itself.

docs/multi-user/README.md

@@ -67,7 +67,14 @@ List all enabled and disabled users.
 mc admin users list myminio
 ```
 
+### 6. Configure `mc`
+```
+mc config host add myminio-newuser http://localhost:9000 newuser newuser123 --api s3v4
+mc cat myminio-newuser/my-bucketname/my-objectname
+```
+
 ## Explore Further
+- [Minio Client Complete Guide](https://docs.minio.io/docs/minio-client-complete-guide)
 - [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide)
 - [Minio Admin Complete Guide](https://docs.minio.io/docs/minio-admin-complete-guide.html)
 - [The Minio documentation website](https://docs.minio.io)

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
  minio1:
-  image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+  image: minio/minio:RELEASE.2018-11-06T01-01-02Z
   volumes:
    - data1:/data
   ports:
@@ -15,7 +15,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio2:
-  image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+  image: minio/minio:RELEASE.2018-11-06T01-01-02Z
   volumes:
    - data2:/data
   ports:
@@ -25,7 +25,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio3:
-  image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+  image: minio/minio:RELEASE.2018-11-06T01-01-02Z
   volumes:
    - data3:/data
   ports:
@@ -35,7 +35,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio4:
-  image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+  image: minio/minio:RELEASE.2018-11-06T01-01-02Z
   volumes:
    - data4:/data
   ports:

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    image: minio/minio:RELEASE.2018-11-06T01-01-02Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
       - access_key
 
   minio2:
-    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    image: minio/minio:RELEASE.2018-11-06T01-01-02Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
       - access_key
 
   minio3:
-    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    image: minio/minio:RELEASE.2018-11-06T01-01-02Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
       - access_key
 
   minio4:
-    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    image: minio/minio:RELEASE.2018-11-06T01-01-02Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    image: minio/minio:RELEASE.2018-11-06T01-01-02Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio2:
-    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    image: minio/minio:RELEASE.2018-11-06T01-01-02Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio3:
-    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    image: minio/minio:RELEASE.2018-11-06T01-01-02Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio4:
-    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    image: minio/minio:RELEASE.2018-11-06T01-01-02Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/kubernetes/README.md

@@ -130,7 +130,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+        image: minio/minio:RELEASE.2018-11-06T01-01-02Z
         args:
         - server
         - /data
@@ -309,7 +309,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+        image: minio/minio:RELEASE.2018-11-06T01-01-02Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data
@@ -520,7 +520,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+        image: minio/minio:RELEASE.2018-11-06T01-01-02Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-distributed-statefulset.yaml

@@ -21,7 +21,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+        image: minio/minio:RELEASE.2018-11-06T01-01-02Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes/minio-gcs-gateway-deployment.yaml

@@ -21,7 +21,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+        image: minio/minio:RELEASE.2018-11-06T01-01-02Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-standalone-deployment.yaml

@@ -29,7 +29,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+        image: minio/minio:RELEASE.2018-11-06T01-01-02Z
         args:
         - server
         - /data

docs/sts/README.md

@@ -11,7 +11,7 @@ Following are advantages for using temporary credentials:
 - Temporary credentials have a limited lifetime, there is no need to rotate them or explicitly revoke them. Expired temporary credentials cannot be reused.
 
 ## Identity Federation
-[**Client grants**](https://github.com/minio/minio/blob/master/docs/sts/client-grants.md) - Let applications request `client_grants` using any well-known third party identity provider such as KeyCloak, WSO2. This is known as the client grants approach to temporary access. Using this approach helps clients keep Minio credentials to be secured. Minio STS client grants supports WSO2, Keycloak.
+[**Client grants**](https://github.com/minio/minio/blob/master/docs/sts/client-grants.md) - Let applications request `client_grants` using any well-known third party identity provider such as KeyCloak, WSO2. This is known as the client grants approach to temporary access. Using this approach helps clients keep Minio credentials to be secured. Minio STS supports client grants, tested against identity providers such as WSO2, KeyCloak.
 
 ## Get started
 In this document we will explain in detail on how to configure all the prerequisites, primarily WSO2, OPA (open policy agent).
@@ -42,7 +42,7 @@ export MINIO_ACCESS_KEY=aws_access_key
 export MINIO_SECRET_KEY=aws_secret_key
 export MINIO_IAM_JWKS_URL=https://localhost:9443/oauth2/jwks
 export MINIO_IAM_OPA_URL=http://localhost:8181/v1/data/httpapi/authz
-export MINIO_ETCD_ENDPOINTS=localhost:2379
+export MINIO_ETCD_ENDPOINTS=http://localhost:2379
 minio gateway s3
 ```
 

docs/sts/client-grants.md

@@ -30,6 +30,31 @@ XML response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](http
 #### Errors
 XML error response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_Errors)
 
+#### Sample Request
+```
+http://minio.cluster:9000?Action=AssumeRoleWithClientGrants&DurationSeconds=3600&Token=eyJ4NXQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJraWQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJhbGciOiJSUzI1NiJ9.eyJhdWQiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiYXpwIjoiUG9FZ1hQNnVWTzQ1SXNFTlJuZ0RYajVBdTVZYSIsImlzcyI6Imh0dHBzOlwvXC9sb2NhbGhvc3Q6OTQ0M1wvb2F1dGgyXC90b2tlbiIsImV4cCI6MTU0MTgwOTU4MiwiaWF0IjoxNTQxODA1OTgyLCJqdGkiOiI2Y2YyMGIwZS1lNGZmLTQzZmQtYTdiYS1kYTc3YTE3YzM2MzYifQ.Jm29jPliRvrK6Os34nSK3rhzIYLFjE__zdVGNng3uGKXGKzP3We_i6NPnhA0szJXMOKglXzUF1UgSz8MctbaxFS8XDusQPVe4LkB_45hwBm6TmBxzui911nt-1RbBLN_jZIlvl2lPrbTUH5hSn9kEkph6seWanTNQpz9tNEoVa6R_OX3kpJqxe8tLQUWw453A1JTwFNhdHa6-f1K8_Q_eEZ_4gOYINQ9t_fhTibdbkXZkJQFLop-Jwoybi9s4nwQU_dATocgcufq5eCeNItQeleT-23lGxIz0X7CiJrJynYLdd-ER0F77SumqEb5iCxhxuf4H7dovwd1kAmyKzLxpw
+```
+
+#### Sample Response
+```
+<?xml version="1.0" encoding="UTF-8"?>
+<AssumeRoleWithClientGrantsResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+  <AssumeRoleWithClientGrantsResult>
+    <AssumedRoleUser>
+      <Arn/>
+      <AssumeRoleId/>
+    </AssumedRoleUser>
+    <Credentials>
+      <AccessKeyId>Y4RJU1RNFGK48LGO9I2S</AccessKeyId>
+      <SecretAccessKey>sYLRKS1Z7hSjluf6gEbb9066hnx315wHTiACPAjg</SecretAccessKey>
+      <Expiration>2018-11-09T16:51:11-08:00</Expiration>
+      <SessionToken>eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJZNFJKVTFSTkZHSzQ4TEdPOUkyUyIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTQxODExMDcxLCJpYXQiOjE1NDE4MDc0NzEsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiYTBiMjc2MjktZWUxYS00M2JmLTg3MzktZjMzNzRhNGNkYmMwIn0.ewHqKVFTaP-j_kgZrcOEKroNUjk10GEp8bqQjxBbYVovV0nHO985VnRESFbcT6XMDDKHZiWqN2vi_ETX_u3Q-w</SessionToken>
+    </Credentials>
+  </AssumeRoleWithClientGrantsResult>
+  <ResponseMetadata/>
+</AssumeRoleWithClientGrantsResponse>
+```
+
 #### Testing
 ```
 $ export MINIO_ACCESS_KEY=minio
@@ -56,7 +81,7 @@ $ mc admin config get myminio
 ```
 
 Testing with an example
-> Obtaining client ID and secrets follow [WSO2 configuring documentation](./wso2.md)
+> Obtaining client ID and secrets follow [WSO2 configuring documentation](https://github.com/minio/minio/blob/master/docs/sts/wso2.md)
 
 ```
 go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOWvhRWL4TUCga

docs/sts/etcd.md

@@ -6,7 +6,7 @@ etcd is a distributed key value store that provides a reliable way to store data
 - Docker 18.03 or above, refer here for [installation](https://docs.docker.com/install/).
 
 ### 2. Start etcd
-etcd uses [gcr.io/etcd-development/etcd](gcr.io/etcd-development/etcd) as a primary container registry.
+etcd uses [gcr.io/etcd-development/etcd](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) as a primary container registry.
 
 ```
 rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \
@@ -29,15 +29,19 @@ rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \
   --initial-cluster-state new
 ```
 
+You may also setup etcd with TLS following this documentation [here](https://coreos.com/etcd/docs/latest/op-guide/security.html)
+
 ### 3. Setup Minio with etcd
 Minio server expects environment variable for etcd as `MINIO_ETCD_ENDPOINTS`, this environment variable takes many comma separated entries.
 ```
-export MINIO_ETCD_ENDPOINTS=localhost:2379
+export MINIO_ETCD_ENDPOINTS=http://localhost:2379
 minio server /data
 ```
 
-### 5. Test with Minio STS API
-Assuming that you have configured Minio server to support STS API by following the doc [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide) and once you have obtained the JWT from WSO2 as mentioned in [WSO2 Quickstart Guide](https://docs.minio.io/docs/wso2-quickstart-guide).
+NOTE: If `etcd` is configured with `Client-to-server authentication with HTTPS client certificates` then you need to use additional envs such as `MINIO_ETCD_CLIENT_CERT` pointing to path to `etcd-client.crt` and `MINIO_ETCD_CLIENT_CERT_KEY` path to `etcd-client.key` .
+
+### 4. Test with Minio STS API
+Assuming that you have configured Minio server to support STS API by following the doc [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide) and once you have obtained the JWT from WSO2 as mentioned in [WSO2 Quickstart Guide](https://github.com/minio/minio/blob/master/docs/sts/wso2.md).
 ```
 go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOWvhRWL4TUCga
 
@@ -50,7 +54,7 @@ go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOW
 }
 ```
 
-These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://docs.minio.io/docs/api-assume-role-with-client-grants).
+These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://github.com/minio/minio/blob/master/docs/sts/client-grants.md).
 
 ## Explore Further
 - [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide)

docs/sts/opa.md

@@ -78,7 +78,7 @@ go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOW
 }
 ```
 
-These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://docs.minio.io/docs/api-assume-role-with-client-grants).
+These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://github.com/minio/minio/blob/master/docs/sts/client-grants.md).
 
 ## Explore Further
 - [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide)

docs/sts/putobject.rego

@@ -0,0 +1,8 @@
+package httpapi.authz
+
+import input as http_api
+
+allow {
+ input.action = "s3:PutObject"
+ input.owner = false
+}

docs/sts/sts.env

@@ -2,3 +2,4 @@ export MINIO_ACCESS_KEY=minio
 export MINIO_SECRET_KEY=minio123
 export MINIO_IAM_JWKS_URL=http://localhost:9763/oauth2/jwks
 export MINIO_IAM_OPA_URL=http://localhost:8181/v1/data/httpapi/authz
+export MINIO_ETCD_ENDPOINTS=http://localhost:2379

docs/sts/wso2.md

@@ -54,15 +54,18 @@ The access token received is a signed JSON Web Token (JWT). Use a JWT decoder to
 
 |Claim Name|Type|Claim Value|
 |:--:|:--:|:--:|
-|iss| string | The issuer of the JWT. The '> Identity Provider Entity Id ' value of the OAuth2/OpenID Connect Inbound Authentication configuration of the Resident Identity Provider is returned here. |
-|aud| string array | The token audience list. The client identifier of the OAuth clients that the JWT is intended for, is sent herewith. |
-|azp| string | The authorized party for which the token is issued to. The client identifier of the OAuth client that the token is issued for, is sent herewith. |
-|iat| integer |	The token issue time. |
-|exp| integer |	The token expiration time. |
-|jti| string | Unique identifier for the JWT token. |
+|iss| _string_ | The issuer of the JWT. The '> Identity Provider Entity Id ' value of the OAuth2/OpenID Connect Inbound Authentication configuration of the Resident Identity Provider is returned here. |
+|aud| _string array_ | The token audience list. The client identifier of the OAuth clients that the JWT is intended for, is sent herewith. |
+|azp| _string_ | The authorized party for which the token is issued to. The client identifier of the OAuth client that the token is issued for, is sent herewith. |
+|iat| _integer_ |	The token issue time. |
+|exp| _integer_ |	The token expiration time. |
+|jti| _string_ | Unique identifier for the JWT token. |
+|policy| _string_ | Canned policy name to be applied for STS credentials. (Optional) |
 
 Using the above `access_token` we can perform an STS request to Minio to get temporary credentials for Minio API operations. Minio STS API uses [JSON Web Key Set Endpoint](https://docs.wso2.com/display/IS541/JSON+Web+Key+Set+Endpoint) to validate if JWT is valid and is properly signed.
 
+Optionally you can also configure `policy` as a custom claim for the JWT service provider follow [here](https://docs.wso2.com/display/IS550/Configuring+Claims+for+a+Service+Provider) and [here](https://docs.wso2.com/display/IS550/Handling+Custom+Claims+with+the+JWT+Bearer+Grant+Type) for relevant docs on how to configure claims for a service provider.
+
 ### 5. Setup Minio with JWKS URL
 Minio server expects environment variable for JWKS url as `MINIO_IAM_JWKS_URL`, this environment variable takes a single entry.
 ```
@@ -83,7 +86,7 @@ go run full-example.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOW
 }
 ```
 
-These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://docs.minio.io/docs/api-assume-role-with-client-grants).
+These credentials can now be used to perform Minio API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://github.com/minio/minio/blob/master/docs/sts/client-grants.md).
 
 ## Explore Further
 - [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide)

docs/zh_CN/gateway/azure.md

@@ -37,11 +37,7 @@ mc ls myazure
 [2017-02-26 22:10:11 PST]     0B test-container1/
 ```
 
-### 已知的限制
-[限制](https://github.com/minio/minio/blob/master/docs/gateway/azure-limitations.md)
-
 ## 了解更多
 - [`mc` 命令行接口](https://docs.minio.io/cn/minio-client-quickstart-guide)
 - [`aws` 命令行接口](https://docs.minio.io/cn/aws-cli-with-minio)
-- [`minfs` 文件系统接口](https://docs.minio.io/cn/minfs-quickstart-guide)
 - [`minio-go` Go SDK](https://docs.minio.io/cn/golang-client-quickstart-guide)

docs/zh_CN/gateway/gcs.md

@@ -52,12 +52,8 @@ mc ls mygcs
 [2017-02-26 22:10:11 PST]     0B test-container1/
 ```
 
-### 已知的限制
-[限制](https://github.com/minio/minio/blob/master/docs/gateway/gcs-limitations.md)
-
 ## 了解更多
 - [`mc` 命令行接口](https://docs.minio.io/cn/minio-client-quickstart-guide)
 - [`aws` 命令行接口](https://docs.minio.io/cn/aws-cli-with-minio)
-- [`minfs` 文件系统接口](https://docs.minio.io/cn/minfs-quickstart-guide)
 - [`minio-go` Go SDK](https://docs.minio.io/cn/golang-client-quickstart-guide)
 

docs/zh_CN/orchestration/kubernetes/README.md

@@ -4,9 +4,7 @@ Kubernetes的部署和状态集提供了在独立，分布式或共享模式下
 
 - Minio [Helm](https://helm.sh) Chart通过一个简单的命令即可提供自定义而且简单的Minio部署。更多关于Minio Helm部署的资料，请访问[这里](#prerequisites).
 
-- 你也可以浏览Kubernetes [Minio示例](https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/README.md) ，通过`.yaml`文件来部署Minio。
-
-- 如果您想在Kubernetes上开始使用Minio，而无需创建真正的容器集群，您也可以使用Minikube [deploy Minio locally](https://raw.githubusercontent.com/minio/minio/master/docs/orchestration/minikube/README.md)。
+- 你也可以浏览Kubernetes [Minio示例](https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes/README.md) ，通过`.yaml`文件来部署Minio。
 
 <a name="prerequisites"></a>
 ## 1. 前提条件

docs/zh_CN/orchestration/minikube/README.md

@@ -1,46 +0,0 @@
-# 使用minikube在本地部署分布式Minio
- [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
-
-Minikube在计算机的VM中运行单节点Kubernetes集群。 这样可以轻松地在计算机上本地运行的Kubernetes上部署分布式Minio服务器。
-
-## 1. 前提条件
-
-本机已经安装[Minikube](https://github.com/kubernetes/minikube/blob/master/README.md#installation) 和 [`kubectl`](https://kubernetes.io/docs/user-guide/prereqs/)
-
-
-## 2. 步骤
-
-* 下载 `minio_distributed.sh` 和 `statefulset.yaml`
-
-```sh
-wget https://raw.githubusercontent.com/minio/minio/master/docs/orchestration/minikube/minio_distributed.sh  
-wget https://raw.githubusercontent.com/minio/minio/master/docs/orchestration/minikube/statefulset.yaml
-```
-
-* 在命令提示符下执行`minio_distributed.sh`脚本。
-
-```sh
-./minio_distributed.sh
-```
-
-脚本执行成功后，您应该会收到一个这样的输出
-
-```sh
-service "minio-public" created
-service "minio" created
-statefulset "minio" created
-```
-这意味着Minio部署在您当地的Minikube安装中。
-
-请注意，服务“minio-public”是一个[clusterIP]（https://kubernetes.io/docs/user-guide/services/#publishing-services---service-types）服务。 它在集群内部IP上暴露服务。 通过`kubectl port-forward`命令连接到Minio实例，执行
-
-```
-kubectl port-forward minio-0 9000:9000
-```
-
-Minio服务器现在可以在`http:// localhost:9000`访问，使用`statefulset.yaml`文件中所述的accessKey和secretKey。
-
-## 3. 注意
-
-Minikube目前不支持动态配置，因此我们手动创建PersistentVolumes（PV）和PersistentVolumeClaims（PVC）。 创建PV和PVC后，我们将调用`statefulset.yaml`配置文件来创建分布式的Minio设置。
-此设置在笔记本电脑/计算机上运行。 因此，只有一个磁盘用作所有minio实例PV的后端。 Minio将这些PV视为单独的磁盘，并报告可用存储不正确。

docs/zh_CN/shared-backend/README.md

@@ -17,7 +17,7 @@ Minio共享模式是为了解决在真实场景中存在的一些问题，而且
 
 ## 1. 前提条件
 
-安装Minio - [Minio快速入门](https://docs.minio.io/cn/minio).
+安装Minio - [Minio快速入门](https://docs.minio.io/cn/).
 
 ## 2. 在共享后端存储上运行Minio
 

pkg/auth/credentials.go

@@ -21,6 +21,7 @@ import (
 	"crypto/subtle"
 	"encoding/base64"
 	"fmt"
+	"strings"
 	"time"
 
 	jwtgo "github.com/dgrijalva/jwt-go"
@@ -60,8 +61,8 @@ func IsAccessKeyValid(accessKey string) bool {
 	return len(accessKey) >= accessKeyMinLen
 }
 
-// isSecretKeyValid - validate secret key for right length.
-func isSecretKeyValid(secretKey string) bool {
+// IsSecretKeyValid - validate secret key for right length.
+func IsSecretKeyValid(secretKey string) bool {
 	return len(secretKey) >= secretKeyMinLen
 }
 
@@ -87,7 +88,7 @@ func (cred Credentials) IsExpired() bool {
 func (cred Credentials) IsValid() bool {
 	// Verify credentials if its enabled or not set.
 	if cred.Status == "enabled" || cred.Status == "" {
-		return IsAccessKeyValid(cred.AccessKey) && isSecretKeyValid(cred.SecretKey) && !cred.IsExpired()
+		return IsAccessKeyValid(cred.AccessKey) && IsSecretKeyValid(cred.SecretKey) && !cred.IsExpired()
 	}
 	return false
 }
@@ -131,7 +132,7 @@ func GetNewCredentialsWithMetadata(m map[string]interface{}, tokenSecret string)
 	if err != nil {
 		return cred, err
 	}
-	cred.SecretKey = string([]byte(base64.URLEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen])
+	cred.SecretKey = strings.Replace(string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen]), "/", "+", -1)
 	cred.Status = "enabled"
 
 	expiry, ok := m["exp"].(float64)
@@ -163,7 +164,7 @@ func CreateCredentials(accessKey, secretKey string) (cred Credentials, err error
 	if !IsAccessKeyValid(accessKey) {
 		return cred, ErrInvalidAccessKeyLength
 	}
-	if !isSecretKeyValid(secretKey) {
+	if !IsSecretKeyValid(secretKey) {
 		return cred, ErrInvalidSecretKeyLength
 	}
 	cred.AccessKey = accessKey

pkg/auth/credentials_test.go

@@ -47,7 +47,7 @@ func TestIsSecretKeyValid(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		result := isSecretKeyValid(testCase.secretKey)
+		result := IsSecretKeyValid(testCase.secretKey)
 		if result != testCase.expectedResult {
 			t.Fatalf("test %v: expected: %v, got: %v", i+1, testCase.expectedResult, result)
 		}