Ignore permission errors on config-dir (#6894)

Fixes #6873

browser/app/js/objects/__tests__/actions.test.js

@@ -26,6 +26,8 @@ jest.mock("../../web", () => ({
     .fn(() => true)
     .mockReturnValueOnce(true)
     .mockReturnValueOnce(false)
+    .mockReturnValueOnce(true)
+    .mockReturnValueOnce(true)
     .mockReturnValueOnce(false),
   ListObjects: jest.fn(({ bucketName }) => {
     if (bucketName === "test-deny") {
@@ -405,7 +407,7 @@ describe("Objects actions", () => {
       store.dispatch(actionsObjects.downloadObject("obj1"))
       const url = `${
         window.location.origin
-      }${minioBrowserPrefix}/download/bk1/${encodeURI("pre1/obj1")}?token=''`
+      }${minioBrowserPrefix}/download/bk1/${encodeURI("pre1/obj1")}?token=`
       expect(setLocation).toHaveBeenCalledWith(url)
     })
 

browser/app/js/objects/actions.js

@@ -16,11 +16,7 @@
 
 import web from "../web"
 import history from "../history"
-import {
-  sortObjectsByName,
-  sortObjectsBySize,
-  sortObjectsByDate
-} from "../utils"
+import { sortObjectsByName, sortObjectsBySize, sortObjectsByDate } from "../utils"
 import { getCurrentBucket } from "../buckets/selectors"
 import { getCurrentPrefix, getCheckedList } from "./selectors"
 import * as alertActions from "../alert/actions"
@@ -60,10 +56,7 @@ export const appendList = (objects, marker, isTruncated) => ({
 
 export const fetchObjects = append => {
   return function(dispatch, getState) {
-    const {
-      buckets: { currentBucket },
-      objects: { currentPrefix, marker }
-    } = getState()
+    const {buckets: {currentBucket}, objects: {currentPrefix, marker}} = getState()
     if (currentBucket) {
       return web
         .ListObjects({
@@ -110,7 +103,7 @@ export const fetchObjects = append => {
 
 export const sortObjects = sortBy => {
   return function(dispatch, getState) {
-    const { objects } = getState()
+    const {objects} = getState()
     const sortOrder = objects.sortBy == sortBy ? !objects.sortOrder : true
     dispatch(setSortBy(sortBy))
     dispatch(setSortOrder(sortOrder))
@@ -210,30 +203,39 @@ export const shareObject = (object, days, hours, minutes) => {
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
     const expiry = days * 24 * 60 * 60 + hours * 60 * 60 + minutes * 60
-    return web
-      .PresignedGet({
-        host: location.host,
-        bucket: currentBucket,
-        object: objectName,
-        expiry
-      })
-      .then(obj => {
-        dispatch(showShareObject(object, obj.url))
-        dispatch(
-          alertActions.set({
-            type: "success",
-            message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
-          })
-        )
-      })
-      .catch(err => {
-        dispatch(
-          alertActions.set({
-            type: "danger",
-            message: err.message
-          })
-        )
-      })
+    if (web.LoggedIn()) {
+      return web
+        .PresignedGet({
+          host: location.host,
+          bucket: currentBucket,
+          object: objectName
+        })
+        .then(obj => {
+          dispatch(showShareObject(object, obj.url))
+          dispatch(
+            alertActions.set({
+              type: "success",
+              message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
+            })
+          )
+        })
+        .catch(err => {
+          dispatch(
+            alertActions.set({
+              type: "danger",
+              message: err.message
+            })
+          )
+        })
+    } else {
+      dispatch(showShareObject(object, `${location.host}` + '/' + `${currentBucket}` + '/' + encodeURI(objectName)))
+      dispatch(
+        alertActions.set({
+          type: "success",
+          message: `Object shared.`
+        })
+      )
+    }
   }
 }
 
@@ -279,7 +281,7 @@ export const downloadObject = object => {
     } else {
       const url = `${
         window.location.origin
-      }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=''`
+      }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
       window.location = url
     }
   }
@@ -308,7 +310,7 @@ export const downloadCheckedObjects = () => {
       objects: getCheckedList(state)
     }
     if (!web.LoggedIn()) {
-      const requestUrl = location.origin + "/minio/zip?token=''"
+      const requestUrl = location.origin + "/minio/zip?token="
       downloadZip(requestUrl, req, dispatch)
     } else {
       return web
@@ -319,14 +321,13 @@ export const downloadCheckedObjects = () => {
           }${minioBrowserPrefix}/zip?token=${res.token}`
           downloadZip(requestUrl, req, dispatch)
         })
-        .catch(err =>
-          dispatch(
-            alertActions.set({
-              type: "danger",
-              message: err.message
-            })
-          )
+        .catch(err => dispatch(
+          alertActions.set({
+            type: "danger",
+            message: err.message
+          })
         )
+      )
     }
   }
 }
@@ -349,8 +350,7 @@ const downloadZip = (url, req, dispatch) => {
       var separator = req.prefix.length > 1 ? "-" : ""
 
       anchor.href = blobUrl
-      anchor.download =
-        req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
+      anchor.download = req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
 
       anchor.click()
       window.URL.revokeObjectURL(blobUrl)

cmd/acl-handlers.go

@@ -57,28 +57,28 @@ type accessControlPolicy struct {
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketACL")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "GetBucketACL", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getBucketACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -92,7 +92,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -106,7 +106,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectACL")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "GetObjectACL", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -114,21 +114,21 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getObjectACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if object exists.
 	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/admin-handlers.go

@@ -578,7 +578,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 				StartTime:     nh.startTime,
 			})
 			if err != nil {
-				writeErrorResponse(w, toAdminAPIErrCode(ctx, err), r.URL)
+				writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 				return
 			}
 			// Client token not specified but a heal sequence exists on a path,
@@ -859,7 +859,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 
 	// Custom IAM policies not allowed for admin user.
 	if accessKey == globalServerConfig.GetCredential().AccessKey {
-		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
 		return
 	}
 
@@ -898,7 +898,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 
 	// Custom IAM policies not allowed for admin user.
 	if accessKey == globalServerConfig.GetCredential().AccessKey {
-		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
 		return
 	}
 

cmd/admin-handlers_test.go

@@ -309,7 +309,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 		for i := 0; i < 10; i++ {
 			objectName := fmt.Sprintf("%s-%d", objName, i)
 			_, err = atb.objLayer.PutObject(context.Background(), bucketName, objectName,
-				mustGetHashReader(t, bytes.NewReader([]byte("hello")),
+				mustGetPutObjReader(t, bytes.NewReader([]byte("hello")),
 					int64(len("hello")), "", ""), nil, ObjectOptions{})
 			if err != nil {
 				t.Fatalf("Failed to create %s - %v", objectName,
@@ -328,7 +328,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 		}
 
 		_, err = atb.objLayer.PutObjectPart(context.Background(), bucketName, objName,
-			uploadID, 3, mustGetHashReader(t, bytes.NewReader(
+			uploadID, 3, mustGetPutObjReader(t, bytes.NewReader(
 				[]byte("hello")), int64(len("hello")), "", ""), ObjectOptions{})
 		if err != nil {
 			t.Fatalf("mp put error: %v", err)

cmd/admin-heal-ops.go

@@ -113,6 +113,32 @@ func initAllHealState(isErasureMode bool) {
 	globalAllHealState = allHealState{
 		healSeqMap: make(map[string]*healSequence),
 	}
+
+	go globalAllHealState.periodicHealSeqsClean()
+}
+
+func (ahs *allHealState) periodicHealSeqsClean() {
+	// Launch clean-up routine to remove this heal sequence (after
+	// it ends) from the global state after timeout has elapsed.
+	ticker := time.NewTicker(time.Minute * 5)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			now := UTCNow()
+			ahs.Lock()
+			for path, h := range ahs.healSeqMap {
+				if h.hasEnded() && h.endTime.Add(keepHealSeqStateDuration).Before(now) {
+					delete(ahs.healSeqMap, path)
+				}
+			}
+			ahs.Unlock()
+		case <-globalServiceDoneCh:
+			// server could be restarting - need
+			// to exit immediately
+			return
+		}
+	}
 }
 
 // getHealSequence - Retrieve a heal sequence by path. The second
@@ -213,41 +239,6 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	// Launch top-level background heal go-routine
 	go h.healSequenceStart()
 
-	// Launch clean-up routine to remove this heal sequence (after
-	// it ends) from the global state after timeout has elapsed.
-	go func() {
-		var keepStateTimeout <-chan time.Time
-		ticker := time.NewTicker(time.Minute)
-		defer ticker.Stop()
-		everyMinute := ticker.C
-		for {
-			select {
-			// Check every minute if heal sequence has ended.
-			case <-everyMinute:
-				if h.hasEnded() {
-					keepStateTimeout = time.After(keepHealSeqStateDuration)
-					everyMinute = nil
-				}
-
-			// This case does not fire until the heal
-			// sequence completes.
-			case <-keepStateTimeout:
-				// Heal sequence has ended, keep
-				// results state duration has elapsed,
-				// so purge state.
-				ahs.Lock()
-				defer ahs.Unlock()
-				delete(ahs.healSeqMap, h.path)
-				return
-
-			case <-globalServiceDoneCh:
-				// server could be restarting - need
-				// to exit immediately
-				return
-			}
-		}
-	}()
-
 	b, err := json.Marshal(madmin.HealStartSuccess{
 		ClientToken:   h.clientToken,
 		ClientAddress: h.clientAddress,
@@ -321,6 +312,9 @@ type healSequence struct {
 	// time at which heal sequence was started
 	startTime time.Time
 
+	// time at which heal sequence has ended
+	endTime time.Time
+
 	// Heal client info
 	clientToken, clientAddress string
 
@@ -498,6 +492,7 @@ func (h *healSequence) healSequenceStart() {
 
 	select {
 	case err, ok := <-h.traverseAndHealDoneCh:
+		h.endTime = UTCNow()
 		h.currentStatus.updateLock.Lock()
 		defer h.currentStatus.updateLock.Unlock()
 		// Heal traversal is complete.
@@ -511,6 +506,7 @@ func (h *healSequence) healSequenceStart() {
 		}
 
 	case <-h.stopSignalCh:
+		h.endTime = UTCNow()
 		h.currentStatus.updateLock.Lock()
 		h.currentStatus.Summary = healStoppedStatus
 		h.currentStatus.FailureDetail = errHealStopSignalled.Error()

cmd/api-datatypes.go

@@ -23,6 +23,8 @@ import (
 const (
 	// Response request id.
 	responseRequestIDKey = "x-amz-request-id"
+	// Deployment id.
+	responseDeploymentIDKey = "x-minio-deployment-id"
 )
 
 // ObjectIdentifier carries key name for the object to delete.

cmd/api-errors.go

@@ -297,6 +297,8 @@ const (
 	ErrEvaluatorInvalidTimestampFormatPatternSymbol
 	ErrEvaluatorBindingDoesNotExist
 	ErrMissingHeaders
+	ErrInvalidColumnIndex
+
 	ErrAdminConfigNotificationTargetsFailed
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
@@ -1431,6 +1433,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Some headers in the query are missing from the file. Check the file and try again.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidColumnIndex: {
+		Code:           "InvalidColumnIndex",
+		Description:    "The column index is invalid. Please check the service documentation and try again.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidDecompressedSize: {
 		Code:           "XMinioInvalidDecompressedSize",
 		Description:    "The data provided is unfit for decompression",
@@ -1658,6 +1665,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrMissingHeaders
 	case format.ErrParseInvalidPathComponent:
 		apiErr = ErrMissingHeaders
+	case format.ErrInvalidColumnIndex:
+		apiErr = ErrInvalidColumnIndex
 	}
 
 	// Compression errors

cmd/api-response.go

@@ -568,13 +568,20 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 }
 
 // writeErrorRespone writes error headers
-func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
+func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL, browser bool) {
 	switch errorCode {
 	case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set("Retry-After", "120")
+	case ErrAccessDenied:
+		if browser {
+			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+			w.WriteHeader(http.StatusTemporaryRedirect)
+			return
+		}
 	}
+
 	apiError := getAPIError(errorCode)
 	// Generate error response.
 	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))

cmd/auth-handler.go

@@ -19,10 +19,10 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/subtle"
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -155,49 +155,66 @@ func getSessionToken(r *http.Request) (token string) {
 	return r.URL.Query().Get("X-Amz-Security-Token")
 }
 
-// Fetch claims in the security token returned by the client and validate the token.
-func getClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
-	stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
-		if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
-			return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
-		}
-		if err := jwtToken.Claims.Valid(); err != nil {
-			return nil, errAuthentication
-		}
-		if claims, ok := jwtToken.Claims.(jwtgo.MapClaims); ok {
-			if _, ok = claims["accessKey"].(string); !ok {
-				return nil, errInvalidAccessKeyID
-			}
-			// JWT token for x-amz-security-token is signed with admin
-			// secret key, temporary credentials become invalid if
-			// server admin credentials change. This is done to ensure
-			// that clients cannot decode the token using the temp
-			// secret keys and generate an entirely new claim by essentially
-			// hijacking the policies. We need to make sure that this is
-			// based an admin credential such that token cannot be decoded
-			// on the client side and is treated like an opaque value.
-			return []byte(globalServerConfig.GetCredential().SecretKey), nil
-		}
-		return nil, errAuthentication
-	}
+// Fetch claims in the security token returned by the client, doesn't return
+// errors - upon errors the returned claims map will be empty.
+func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
+	claims, _ := getClaimsFromToken(r)
+	return claims
+}
+
+// Fetch claims in the security token returned by the client.
+func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 	claims := make(map[string]interface{})
 	token := getSessionToken(r)
 	if token == "" {
-		return nil, ErrNone
+		return claims, nil
 	}
+	stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
+		// JWT token for x-amz-security-token is signed with admin
+		// secret key, temporary credentials become invalid if
+		// server admin credentials change. This is done to ensure
+		// that clients cannot decode the token using the temp
+		// secret keys and generate an entirely new claim by essentially
+		// hijacking the policies. We need to make sure that this is
+		// based an admin credential such that token cannot be decoded
+		// on the client side and is treated like an opaque value.
+		return []byte(globalServerConfig.GetCredential().SecretKey), nil
+	}
+	p := &jwtgo.Parser{
+		ValidMethods: []string{
+			jwtgo.SigningMethodHS256.Alg(),
+			jwtgo.SigningMethodHS512.Alg(),
+		},
+	}
+	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
+	if err != nil {
+		return nil, err
+	}
+	if !jtoken.Valid {
+		return nil, errAuthentication
+	}
+	v, ok := claims["accessKey"]
+	if !ok {
+		return nil, errInvalidAccessKeyID
+	}
+	if _, ok = v.(string); !ok {
+		return nil, errInvalidAccessKeyID
+	}
+	return claims, nil
+}
+
+// Fetch claims in the security token returned by the client and validate the token.
+func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
+	token := getSessionToken(r)
 	if token != "" && cred.AccessKey == "" {
 		return nil, ErrNoAccessKey
 	}
-	if token != cred.SessionToken {
+	if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
 		return nil, ErrInvalidToken
 	}
-	p := &jwtgo.Parser{}
-	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
+	claims, err := getClaimsFromToken(r)
 	if err != nil {
-		return nil, toAPIErrorCode(context.Background(), errAuthentication)
-	}
-	if !jtoken.Valid {
-		return nil, toAPIErrorCode(context.Background(), errAuthentication)
+		return nil, toAPIErrorCode(context.Background(), err)
 	}
 	return claims, ErrNone
 }
@@ -233,6 +250,11 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		return s3Err
 	}
 
+	claims, s3Err := checkClaimsFromToken(r, cred)
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
 	// LocationConstraint is valid only for CreateBucketAction.
 	var locationConstraint string
 	if action == policy.CreateBucketAction {
@@ -256,11 +278,6 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
 
-	claims, s3Err := getClaimsFromToken(r, cred)
-	if s3Err != ErrNone {
-		return s3Err
-	}
-
 	if cred.AccessKey == "" {
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
@@ -399,7 +416,7 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		a.handler.ServeHTTP(w, r)
 		return
 	}
-	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
+	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL, guessIsBrowserReq(r))
 }
 
 // isPutAllowed - check if PUT operation is allowed on the resource, this
@@ -421,7 +438,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
 		return s3Err
 	}
 
-	claims, s3Err := getClaimsFromToken(r, cred)
+	claims, s3Err := checkClaimsFromToken(r, cred)
 	if s3Err != ErrNone {
 		return s3Err
 	}

cmd/benchmark-utils_test.go

@@ -61,7 +61,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < b.N; i++ {
 		// insert the object.
 		objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -123,7 +123,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			md5hex = getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
-				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts)
+				mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts)
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -204,7 +204,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -304,7 +304,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		for pb.Next() {
 			// insert the object.
 			objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+				mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -344,7 +344,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}

cmd/bucket-handlers-listobjects.go

@@ -59,19 +59,19 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -81,14 +81,14 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
 
 	if errCode != ErrNone {
-		writeErrorResponse(w, errCode, r.URL)
+		writeErrorResponse(w, errCode, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
 	if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	listObjectsV2 := objectAPI.ListObjectsV2
@@ -100,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	// marshaled into S3 compatible XML header.
 	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -110,15 +110,16 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 			// Read the decompressed size from the meta.json.
 			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
 			if actualSize < 0 {
-				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
+				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
 				return
 			}
 			// Set the info.Size to the actualSize.
 			listObjectsV2Info.Objects[i].Size = actualSize
 		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
 			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
@@ -140,37 +141,37 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "ListObjectsV1", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Extract all the litsObjectsV1 query params to their native values.
 	prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
 	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
 	// does not throw an error.
 	if maxKeys < 0 {
-		writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
+		writeErrorResponse(w, ErrInvalidMaxKeys, r.URL, guessIsBrowserReq(r))
 		return
 	} // Validate all the query params before beginning to serve the request.
 	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	listObjects := objectAPI.ListObjects
@@ -182,7 +183,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -192,15 +193,16 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 			// Read the decompressed size from the meta.json.
 			actualSize = listObjectsInfo.Objects[i].GetActualSize()
 			if actualSize < 0 {
-				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
+				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
 				return
 			}
 			// Set the info.Size to the actualSize.
 			listObjectsInfo.Objects[i].Size = actualSize
 		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
 			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}

cmd/bucket-handlers.go

@@ -89,19 +89,19 @@ func initFederatorBackend(objLayer ObjectLayer) {
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLocation")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "GetBucketLocation", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -110,7 +110,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -139,42 +139,42 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "ListMultipartUploads", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
 	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	if maxUploads < 0 {
-		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
+		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	if keyMarker != "" {
 		// Marker not common with prefix is not implemented.
 		if !hasPrefix(keyMarker, prefix) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL)
+			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// generate response
@@ -192,11 +192,11 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBuckets")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "ListBuckets", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	listBuckets := objectAPI.ListBuckets
@@ -206,7 +206,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// If etcd, dns federation configured list buckets from etcd.
@@ -214,7 +214,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	if globalDNSConfig != nil {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && err != dns.ErrNoEntriesFound {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 		bucketSet := set.NewStringSet()
@@ -233,7 +233,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		var err error
 		bucketsInfo, err = listBuckets(ctx)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -250,14 +250,14 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteMultipleObjects")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "DeleteMultipleObjects", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -266,7 +266,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		// In the event access is denied, a 200 response should still be returned
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 		if s3Error != ErrAccessDenied {
-			writeErrorResponse(w, s3Error, r.URL)
+			writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -274,14 +274,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Content-Md5 is requied should be set
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(w, ErrMissingContentMD5, r.URL)
+		writeErrorResponse(w, ErrMissingContentMD5, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -297,7 +297,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrInternalError, r.URL)
+		writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -305,7 +305,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedXML, r.URL)
+		writeErrorResponse(w, ErrMalformedXML, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -313,7 +313,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	if globalWORMEnabled {
 		// Not required to check whether given objects exist or not, because
 		// DeleteMultipleObject is always successful irrespective of object existence.
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+		writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -397,11 +397,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucket")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "PutBucket", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -409,21 +409,21 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Validate if location sent by the client is valid, reject
 	// requests which do not follow valid region requirements.
 	if !isValidLocation(location) {
-		writeErrorResponse(w, ErrInvalidRegion, r.URL)
+		writeErrorResponse(w, ErrInvalidRegion, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -432,12 +432,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
-					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 				if err = globalDNSConfig.Put(bucket); err != nil {
 					objectAPI.DeleteBucket(ctx, bucket)
-					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 
@@ -447,18 +447,18 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				writeSuccessResponseHeadersOnly(w)
 				return
 			}
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 
 		}
-		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL)
+		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -475,11 +475,11 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PostPolicyBucket")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "PostPolicyBucket", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -488,17 +488,17 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
 	if err != nil {
-		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		writeErrorResponse(w, ErrInvalidRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// Make sure that the URL  does not contain object name.
 	if bucket != filepath.Clean(resource[1:]) {
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+		writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -507,7 +507,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	reader, err := r.MultipartReader()
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -515,7 +515,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	form, err := reader.ReadForm(maxFormMemory)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -526,13 +526,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if file is provided, error out otherwise.
 	if fileBody == nil {
-		writeErrorResponse(w, ErrPOSTFileRequired, r.URL)
+		writeErrorResponse(w, ErrPOSTFileRequired, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -554,7 +554,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if successRedirect != "" {
 		redirectURL, err = url.Parse(successRedirect)
 		if err != nil {
-			writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+			writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -562,25 +562,25 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Verify policy signature.
 	apiErr := doesPolicySignatureMatch(formValues)
 	if apiErr != ErrNone {
-		writeErrorResponse(w, apiErr, r.URL)
+		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
+		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Make sure formValues adhere to policy restrictions.
 	if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
-		writeErrorResponse(w, apiErr, r.URL)
+		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -589,12 +589,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	lengthRange := postPolicyForm.Conditions.ContentLengthRange
 	if lengthRange.Valid {
 		if fileSize < lengthRange.Min {
-			writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
-			writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -603,17 +603,19 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	metadata := make(map[string]string)
 	err = extractMetadataFromMap(ctx, formValues, metadata)
 	if err != nil {
-		writeErrorResponse(w, ErrInternalError, r.URL)
+		writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
+	rawReader := hashReader
+	pReader := NewPutObjReader(rawReader, nil, nil)
+	var objectEncryptionKey []byte
 	if objectAPI.IsEncryptionSupported() {
 		if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C and SSE-S3 requests
 			var reader io.Reader
@@ -621,27 +623,28 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			if crypto.SSEC.IsRequested(formValues) {
 				key, err = ParseSSECustomerHeader(formValues)
 				if err != nil {
-					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 			}
-			reader, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
+			reader, objectEncryptionKey, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			info := ObjectInfo{Size: fileSize}
 			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
+			pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
 		}
 	}
 
-	objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata, ObjectOptions{})
+	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, metadata, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -700,7 +703,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadBucket")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "HeadBucket", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -732,19 +735,19 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucket")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "DeleteBucket", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -754,7 +757,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 	// Attempt to delete bucket.
 	if err := deleteBucket(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -766,7 +769,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		if err := globalDNSConfig.Delete(bucket); err != nil {
 			// Deleting DNS entry failed, attempt to create the bucket again.
 			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}

cmd/bucket-handlers_test.go

@@ -625,7 +625,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)

cmd/bucket-notification-handlers.go

@@ -44,30 +44,30 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketNotification")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "GetBucketNotification", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -76,7 +76,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 	if err != nil {
 		// Ignore errNoSuchNotifications to comply with AWS S3.
 		if err != errNoSuchNotifications {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -90,7 +90,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	notificationBytes, err := xml.Marshal(nConfig)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -102,16 +102,16 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketNotification")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "PutBucketNotification", mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objectAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -119,19 +119,19 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// PutBucketNotification always needs a Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -143,12 +143,12 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 			apiErr = toAPIErrorCode(ctx, err)
 		}
 
-		writeErrorResponse(w, apiErr, r.URL)
+		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -164,16 +164,16 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListenBucketNotification")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "ListenBucketNotification", mustGetClaimsFromToken(r))
 
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -181,7 +181,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -189,11 +189,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var prefix string
 	if len(values["prefix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNamePrefix, r.URL)
+		writeErrorResponse(w, ErrFilterNamePrefix, r.URL, guessIsBrowserReq(r))
 	}
 	if len(values["prefix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -202,11 +202,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var suffix string
 	if len(values["suffix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNameSuffix, r.URL)
+		writeErrorResponse(w, ErrFilterNameSuffix, r.URL, guessIsBrowserReq(r))
 	}
 	if len(values["suffix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -219,7 +219,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	for _, s := range values["events"] {
 		eventName, err := event.ParseName(s)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -227,19 +227,19 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 
 	if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	host, err := xnet.ParseHost(r.RemoteAddr)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	target, err := target.NewHTTPClientTarget(*host, w)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -247,7 +247,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
@@ -255,13 +255,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -271,7 +271,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 }

cmd/bucket-policy-handlers.go

@@ -40,11 +40,11 @@ const (
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketPolicy")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "PutBucketPolicy", mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -52,43 +52,43 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is missing.
 	// PutBucketPolicy always needs Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL)
+		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is beyond allowed size.
 	if r.ContentLength > maxBucketPolicySize {
-		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
+		writeErrorResponse(w, ErrEntityTooLarge, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Version in policy must not be empty
 	if bucketPolicy.Version == "" {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -103,11 +103,11 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketPolicy")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "DeleteBucketPolicy", mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -115,18 +115,18 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -141,11 +141,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketPolicy")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "GetBucketPolicy", mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -153,26 +153,26 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Read bucket access policy.
 	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/certs.go

@@ -71,9 +71,13 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	// Get all CA file names.
 	var caFiles []string
 	fis, err := readDir(certsCAsDir)
-	if err != nil {
+	if err != nil && err != errFileNotFound {
 		return nil, err
 	}
+	// Return success if CA's directory is missing.
+	if err == errFileNotFound {
+		return nil, nil
+	}
 	for _, fi := range fis {
 		// Skip all directories.
 		if hasSuffix(fi, slashSeparator) {

cmd/certs_test.go

@@ -223,7 +223,8 @@ func TestGetRootCAs(t *testing.T) {
 		certCAsDir  string
 		expectedErr error
 	}{
-		{"nonexistent-dir", errFileNotFound},
+		// ignores non-existent directories.
+		{"nonexistent-dir", nil},
 		// Ignores directories.
 		{dir1, nil},
 		// Ignore empty directory.

cmd/common-main.go

@@ -32,6 +32,8 @@ import (
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/target/console"
+	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
 	xnet "github.com/minio/minio/pkg/net"
@@ -54,25 +56,25 @@ func loadLoggers() {
 	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
 	if ok {
 		// Enable audit HTTP logging through ENV.
-		logger.AddAuditTarget(logger.NewHTTP(auditEndpoint, NewCustomHTTPTransport()))
+		logger.AddAuditTarget(http.New(auditEndpoint, NewCustomHTTPTransport()))
 	}
 
 	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
 	if ok {
 		// Enable HTTP logging through ENV.
-		logger.AddTarget(logger.NewHTTP(loggerEndpoint, NewCustomHTTPTransport()))
+		logger.AddTarget(http.New(loggerEndpoint, NewCustomHTTPTransport()))
 	} else {
 		for _, l := range globalServerConfig.Logger.HTTP {
 			if l.Enabled {
 				// Enable http logging
-				logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
+				logger.AddTarget(http.New(l.Endpoint, NewCustomHTTPTransport()))
 			}
 		}
 	}
 
 	if globalServerConfig.Logger.Console.Enabled {
 		// Enable console logging
-		logger.AddTarget(logger.NewConsole())
+		logger.AddTarget(console.New())
 	}
 
 }

cmd/config-common.go

@@ -70,7 +70,7 @@ func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data
 		return err
 	}
 
-	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, hashReader, nil, ObjectOptions{})
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), nil, ObjectOptions{})
 	return err
 }
 

cmd/config-current.go

@@ -556,8 +556,10 @@ func (s *serverConfig) loadToCachedConfigs() {
 	if globalPolicyOPA == nil {
 		if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
 			globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
-				URL:       s.Policy.OPA.URL,
-				AuthToken: s.Policy.OPA.AuthToken,
+				URL:         s.Policy.OPA.URL,
+				AuthToken:   s.Policy.OPA.AuthToken,
+				Transport:   NewCustomHTTPTransport(),
+				CloseRespFn: CloseResponse,
 			})
 		}
 	}

cmd/config-dir.go

@@ -74,7 +74,15 @@ func (config *ConfigDir) GetCADir() string {
 
 // Create - creates configuration directory tree.
 func (config *ConfigDir) Create() error {
-	return os.MkdirAll(config.GetCADir(), 0700)
+	err := os.MkdirAll(config.GetCADir(), 0700)
+	// It is possible in kubernetes like deployments this directory
+	// is already mounted and is not writable, ignore any write errors.
+	if err != nil {
+		if os.IsPermission(err) {
+			err = nil
+		}
+	}
+	return err
 }
 
 // GetMinioConfigFile - returns absolute path of config.json file.

cmd/config-migrate.go

@@ -2413,8 +2413,64 @@ func migrateV27ToV28() error {
 	return nil
 }
 
+// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
+func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
+	defer func() {
+		// Rename config.json to config.json.deprecated only upon
+		// success of this function.
+		if err == nil {
+			os.Rename(getConfigFile(), getConfigFile()+".deprecated")
+		}
+	}()
+
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+	// Construct path to config.json for the given bucket.
+	transactionConfigFile := configFile + ".transaction"
+
+	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
+	// and configFile, take a transaction lock to avoid data race between readConfig()
+	// and saveConfig().
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	if err = objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
+	defer objLock.Unlock()
+
+	// Verify if backend already has the file.
+	if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
+		return err
+	} // if errConfigNotFound proceed to migrate..
+
+	var config = &serverConfig{}
+	if _, err = Load(getConfigFile(), config); err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+		// Read from deprecate file as well if necessary.
+		if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
+			return err
+		}
+	}
+
+	return saveServerConfig(context.Background(), objAPI, config)
+}
+
 // Migrates '.minio.sys/config.json' to v32.
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	// Construct path to config.json for the given bucket.
+	transactionConfigFile := configFile + ".transaction"
+
+	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
+	// and configFile, take a transaction lock to avoid data race between readConfig()
+	// and saveConfig().
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	if err := objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
+	defer objLock.Unlock()
+
 	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
 		return err
 	}

cmd/config.go

@@ -156,30 +156,6 @@ func NewConfigSys() *ConfigSys {
 	return &ConfigSys{}
 }
 
-// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
-func migrateConfigToMinioSys(objAPI ObjectLayer) error {
-	defer os.Rename(getConfigFile(), getConfigFile()+".deprecated")
-
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	// Verify if backend already has the file.
-	if err := checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
-		return err
-	} // if errConfigNotFound proceed to migrate..
-
-	var config = &serverConfig{}
-	if _, err := Load(getConfigFile(), config); err != nil {
-		if !os.IsNotExist(err) {
-			return err
-		}
-		// Read from deprecate file as well if necessary.
-		if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
-			return err
-		}
-	}
-
-	return saveServerConfig(context.Background(), objAPI, config)
-}
-
 // Initialize and load config from remote etcd or local config directory
 func initConfig(objAPI ObjectLayer) error {
 	if objAPI == nil {

cmd/copy-part-range.go

@@ -22,16 +22,16 @@ import (
 )
 
 // Writes S3 compatible copy part range error.
-func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL) {
+func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL, browser bool) {
 	switch err {
 	case errInvalidRange:
-		writeErrorResponse(w, ErrInvalidCopyPartRange, url)
+		writeErrorResponse(w, ErrInvalidCopyPartRange, url, browser)
 		return
 	case errInvalidRangeSource:
-		writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url)
+		writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url, browser)
 		return
 	default:
-		writeErrorResponse(w, ErrInternalError, url)
+		writeErrorResponse(w, ErrInternalError, url, browser)
 		return
 	}
 }

cmd/crypto/vault.go

@@ -192,6 +192,11 @@ func NewVault(kmsConf KMSConfig) (KMS, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	if ns, ok := os.LookupEnv("VAULT_NAMESPACE"); ok {
+		c.SetNamespace(ns)
+	}
+
 	accessToken, leaseDuration, err := getVaultAccessToken(c, config.Auth.AppRole.ID, config.Auth.AppRole.Secret)
 	if err != nil {
 		return nil, err

cmd/disk-cache-fs.go

@@ -30,7 +30,6 @@ import (
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -258,7 +257,7 @@ func (cfs *cacheFSObjects) IsOnline() bool {
 }
 
 // Caches the object to disk
-func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) error {
+func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) error {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:
@@ -301,7 +300,8 @@ func (cfs *cacheFSObjects) Exists(ctx context.Context, bucket, object string) bo
 
 // Identical to fs PutObject operation except that it uses ETag in metadata
 // headers.
-func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+	data := r.Reader
 	fs := cfs.FSObjects
 	// Lock the object.
 	objectLock := fs.nsMutex.NewNSLock(bucket, object)

cmd/disk-cache.go

@@ -60,16 +60,16 @@ type cacheObjects struct {
 	GetObjectNInfoFn          func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfoFn           func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObjectFn               func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn            func(ctx context.Context, bucket, object string) error
 	ListObjectsFn             func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2Fn           func(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
 	ListBucketsFn             func(ctx context.Context) (buckets []BucketInfo, err error)
 	GetBucketInfoFn           func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
 	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
-	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
+	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUploadFn    func(ctx context.Context, bucket, object, uploadID string) error
-	CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
+	CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteBucketFn            func(ctx context.Context, bucket string) error
 }
 
@@ -92,14 +92,14 @@ type CacheObjectLayer interface {
 	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 
 	// Multipart operations.
 	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
-	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
+	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
-	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
+	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
 
 	// Storage operations.
 	StorageInfo(ctx context.Context) CacheStorageInfo
@@ -247,7 +247,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 
 	go func() {
 		opts := ObjectOptions{}
-		putErr := dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(bkReader.ObjInfo), opts)
+		putErr := dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(bkReader.ObjInfo), opts)
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader
 		pipeWriter.CloseWithError(putErr)
@@ -303,7 +303,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		}
 		dcache.Delete(ctx, bucket, object)
 	}
-	if startOffset != 0 || length != objInfo.Size {
+	if startOffset != 0 || (length > 0 && length != objInfo.Size) {
 		// We don't cache partial objects.
 		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
@@ -320,7 +320,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		gerr := GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts)
 		pipeWriter.CloseWithError(gerr) // Close writer explicitly signaling we wrote all data.
 	}()
-	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo), opts)
+	err = dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(objInfo), opts)
 	if err != nil {
 		return err
 	}
@@ -644,8 +644,9 @@ func (c cacheObjects) isCacheExclude(bucket, object string) bool {
 }
 
 // PutObject - caches the uploaded object for single Put operations
-func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	putObjectFn := c.PutObjectFn
+	data := r.Reader
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
@@ -666,20 +667,20 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 	objInfo = ObjectInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, r.MD5HexString(), r.SHA256HexString(), r.ActualSize())
+	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
-	cHashReader, err := hash.NewReader(rPipe, size, r.MD5HexString(), r.SHA256HexString(), r.ActualSize())
+	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	oinfoCh := make(chan ObjectInfo)
 	errCh := make(chan error)
 	go func() {
-		oinfo, perr := putObjectFn(ctx, bucket, object, hashReader, metadata, opts)
+		oinfo, perr := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), metadata, opts)
 		if perr != nil {
 			pipeWriter.CloseWithError(perr)
 			wPipe.CloseWithError(perr)
@@ -692,14 +693,14 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 	}()
 
 	go func() {
-		if err = dcache.Put(ctx, bucket, object, cHashReader, metadata, opts); err != nil {
+		if err = dcache.Put(ctx, bucket, object, NewPutObjReader(cHashReader, nil, nil), metadata, opts); err != nil {
 			wPipe.CloseWithError(err)
 			return
 		}
 	}()
 
 	mwriter := io.MultiWriter(pipeWriter, wPipe)
-	_, err = io.Copy(mwriter, r)
+	_, err = io.Copy(mwriter, data)
 	if err != nil {
 		err = <-errCh
 		return objInfo, err
@@ -734,16 +735,17 @@ func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object str
 }
 
 // PutObjectPart - uploads part to backend and cache simultaneously.
-func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
+func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
+	data := r.Reader
 	putObjectPartFn := c.PutObjectPartFn
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, r, opts)
 	}
 
 	if c.isCacheExclude(bucket, object) {
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, r, opts)
 	}
 
 	// make sure cache has at least size space available
@@ -753,7 +755,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 		case dcache.purgeChan <- struct{}{}:
 		default:
 		}
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, r, opts)
 	}
 
 	info = PartInfo{}
@@ -772,7 +774,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 	pinfoCh := make(chan PartInfo)
 	errorCh := make(chan error)
 	go func() {
-		info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, hashReader, opts)
+		info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, NewPutObjReader(hashReader, nil, nil), opts)
 		if err != nil {
 			close(pinfoCh)
 			pipeWriter.CloseWithError(err)
@@ -784,7 +786,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 		pinfoCh <- info
 	}()
 	go func() {
-		if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, cHashReader, opts); perr != nil {
+		if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, NewPutObjReader(cHashReader, nil, nil), opts); perr != nil {
 			wPipe.CloseWithError(perr)
 			return
 		}
@@ -826,25 +828,25 @@ func (c cacheObjects) AbortMultipartUpload(ctx context.Context, bucket, object,
 }
 
 // CompleteMultipartUpload - completes multipart upload operation on backend and cache.
-func (c cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
+func (c cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	completeMultipartUploadFn := c.CompleteMultipartUploadFn
 
 	if c.isCacheExclude(bucket, object) {
-		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts)
+		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
 
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts)
+		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
 	// perform backend operation
-	objInfo, err = completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts)
+	objInfo, err = completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
 	if err != nil {
 		return
 	}
 	// create new multipart upload in cache with same uploadID
-	dcache.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts)
+	dcache.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	return
 }
 
@@ -969,7 +971,7 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		},
-		PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts)
 		},
 		DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
@@ -990,14 +992,14 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
 			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata, opts)
 		},
-		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
+		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
 			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 		},
 		AbortMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string) error {
 			return newObjectLayerFn().AbortMultipartUpload(ctx, bucket, object, uploadID)
 		},
-		CompleteMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
-			return newObjectLayerFn().CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts)
+		CompleteMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 		},
 		DeleteBucketFn: func(ctx context.Context, bucket string) error {
 			return newObjectLayerFn().DeleteBucket(ctx, bucket)

cmd/disk-cache_test.go

@@ -199,7 +199,7 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
+	err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -275,12 +275,12 @@ func TestDiskCacheMaxUse(t *testing.T) {
 		t.Fatal(err)
 	}
 	if !cache.diskAvailable(int64(size)) {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
+		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
 		if err != errDiskFull {
 			t.Fatal("Cache max-use limit violated.")
 		}
 	} else {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
+		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/dummy-object-layer_test.go

@@ -21,7 +21,6 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -72,7 +71,7 @@ func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object s
 	return
 }
 
-func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 
@@ -96,7 +95,7 @@ func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcO
 	return
 }
 
-func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
+func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
 	return
 }
 
@@ -108,7 +107,7 @@ func (api *DummyObjectLayer) AbortMultipartUpload(ctx context.Context, bucket, o
 	return
 }
 
-func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 

cmd/encryption-v1.go

@@ -22,6 +22,7 @@ import (
 	"crypto/rand"
 	"crypto/subtle"
 	"encoding/binary"
+	"encoding/hex"
 	"errors"
 	"io"
 	"net/http"
@@ -205,21 +206,20 @@ func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]s
 	sealedKey = objectKey.Seal(extKey, crypto.GenerateIV(rand.Reader), crypto.SSEC.String(), bucket, object)
 	crypto.SSEC.CreateMetadata(metadata, sealedKey)
 	return objectKey[:], nil
-
 }
 
-func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string, sseS3 bool) (io.Reader, error) {
+func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string, sseS3 bool) (r io.Reader, encKey []byte, err error) {
 	objectEncryptionKey, err := newEncryptMetadata(key, bucket, object, metadata, sseS3)
 	if err != nil {
-		return nil, err
+		return nil, encKey, err
 	}
 
 	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey[:], MinVersion: sio.Version20})
 	if err != nil {
-		return nil, crypto.ErrInvalidCustomerKey
+		return nil, encKey, crypto.ErrInvalidCustomerKey
 	}
 
-	return reader, nil
+	return reader, objectEncryptionKey, nil
 }
 
 // set new encryption metadata from http request headers for SSE-C and generated key from KMS in the case of
@@ -241,19 +241,18 @@ func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[
 // EncryptRequest takes the client provided content and encrypts the data
 // with the client provided key. It also marks the object as client-side-encrypted
 // and sets the correct headers.
-func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (io.Reader, error) {
+func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (reader io.Reader, objEncKey []byte, err error) {
 
 	var (
 		key []byte
-		err error
 	)
 	if crypto.S3.IsRequested(r.Header) && crypto.SSEC.IsRequested(r.Header) {
-		return nil, crypto.ErrIncompatibleEncryptionMethod
+		return nil, objEncKey, crypto.ErrIncompatibleEncryptionMethod
 	}
 	if crypto.SSEC.IsRequested(r.Header) {
 		key, err = ParseSSECustomerRequest(r)
 		if err != nil {
-			return nil, err
+			return nil, objEncKey, err
 		}
 	}
 	return newEncryptReader(content, key, bucket, object, metadata, crypto.S3.IsRequested(r.Header))
@@ -578,7 +577,6 @@ func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
 
 		d.partDecRelOffset += int64(n1)
 	}
-
 	return len(p), nil
 }
 
@@ -813,7 +811,6 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 
 // getEncryptedMultipartsOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
 func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (uint32, int64, int64) {
-
 	// Calculate encrypted offset of a multipart object
 	computeEncOffset := func(off int64, obj ObjectInfo) (seqNumber uint32, encryptedOffset int64, err error) {
 		var curPartEndOffset uint64
@@ -909,6 +906,64 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	return size, nil
 }
 
+// For encrypted objects, the ETag sent by client if available
+// is stored in encrypted form in the backend. Decrypt the ETag
+// if ETag was previously encrypted.
+func getDecryptedETag(headers http.Header, objInfo ObjectInfo, copySource bool) (decryptedETag string) {
+	var (
+		key [32]byte
+		err error
+	)
+	// If ETag is contentMD5Sum return it as is.
+	if len(objInfo.ETag) == 32 {
+		return objInfo.ETag
+	}
+
+	if crypto.IsMultiPart(objInfo.UserDefined) {
+		return objInfo.ETag
+	}
+	if crypto.SSECopy.IsRequested(headers) {
+		key, err = crypto.SSECopy.ParseHTTP(headers)
+		if err != nil {
+			return objInfo.ETag
+		}
+	}
+	// As per AWS S3 Spec, ETag for SSE-C encrypted objects need not be MD5Sum of the data.
+	// Since server side copy with same source and dest just replaces the ETag, we save
+	// encrypted content MD5Sum as ETag for both SSE-C and SSE-S3, we standardize the ETag
+	//encryption across SSE-C and SSE-S3, and only return last 32 bytes for SSE-C
+	if crypto.SSEC.IsEncrypted(objInfo.UserDefined) && !copySource {
+		return objInfo.ETag[len(objInfo.ETag)-32:]
+	}
+
+	objectEncryptionKey, err := decryptObjectInfo(key[:], objInfo.Bucket, objInfo.Name, objInfo.UserDefined)
+	if err != nil {
+		return objInfo.ETag
+	}
+	return tryDecryptETag(objectEncryptionKey, objInfo.ETag, false)
+}
+
+// helper to decrypt Etag given object encryption key and encrypted ETag
+func tryDecryptETag(key []byte, encryptedETag string, ssec bool) string {
+	// ETag for SSE-C encrypted objects need not be content MD5Sum.While encrypted
+	// md5sum is stored internally, return just the last 32 bytes of hex-encoded and
+	// encrypted md5sum string for SSE-C
+	if ssec {
+		return encryptedETag[len(encryptedETag)-32:]
+	}
+	var objectKey crypto.ObjectKey
+	copy(objectKey[:], key)
+	encBytes, err := hex.DecodeString(encryptedETag)
+	if err != nil {
+		return encryptedETag
+	}
+	etagBytes, err := objectKey.UnsealETag(encBytes)
+	if err != nil {
+		return encryptedETag
+	}
+	return hex.EncodeToString(etagBytes)
+}
+
 // GetDecryptedRange - To decrypt the range (off, length) of the
 // decrypted object stream, we need to read the range (encOff,
 // encLength) of the encrypted object stream to decrypt it, and
@@ -1075,7 +1130,7 @@ func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErr
 // decryption succeeded.
 //
 // DecryptObjectInfo also returns whether the object is encrypted or not.
-func DecryptObjectInfo(info ObjectInfo, headers http.Header) (encrypted bool, err error) {
+func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, err error) {
 	// Directories are never encrypted.
 	if info.IsDir {
 		return false, nil
@@ -1094,6 +1149,11 @@ func DecryptObjectInfo(info ObjectInfo, headers http.Header) (encrypted bool, er
 			return
 		}
 		_, err = info.DecryptedSize()
+
+		if crypto.IsEncrypted(info.UserDefined) && !crypto.IsMultiPart(info.UserDefined) {
+			info.ETag = getDecryptedETag(headers, *info, false)
+		}
+
 	}
 	return
 }

cmd/encryption-v1_test.go

@@ -139,7 +139,7 @@ func TestEncryptRequest(t *testing.T) {
 		for k, v := range test.header {
 			req.Header.Set(k, v)
 		}
-		_, err := EncryptRequest(content, req, "bucket", "object", test.metadata)
+		_, _, err := EncryptRequest(content, req, "bucket", "object", test.metadata)
 
 		if err != nil {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
@@ -324,7 +324,7 @@ var decryptObjectInfoTests = []struct {
 
 func TestDecryptObjectInfo(t *testing.T) {
 	for i, test := range decryptObjectInfoTests {
-		if encrypted, err := DecryptObjectInfo(test.info, test.headers); err != test.expErr {
+		if encrypted, err := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
 			t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
 		} else if enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
 			t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)

cmd/format-fs.go

@@ -24,7 +24,6 @@ import (
 	"path"
 	"time"
 
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -273,7 +272,7 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			rlk.Close()
 			return nil, err
 		}
-		logger.SetDeploymentID(id)
+		globalDeploymentID = id
 		return rlk, nil
 	}
 }

cmd/format-xl.go

@@ -381,6 +381,7 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
 
 var ignoredHiddenDirectories = []string{
 	minioMetaBucket,
+	".snapshot",
 	"lost+found",
 	"$RECYCLE.BIN",
 	"System Volume Information",

cmd/fs-v1-metadata_test.go

@@ -54,7 +54,7 @@ func TestReadFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
@@ -89,7 +89,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 

cmd/fs-v1-multipart.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -31,8 +30,6 @@ import (
 
 	"github.com/minio/minio/cmd/logger"
 	mioutil "github.com/minio/minio/pkg/ioutil"
-
-	"github.com/minio/minio/pkg/hash"
 )
 
 // Returns EXPORT/.minio.sys/multipart/SHA256/UPLOADID
@@ -259,7 +256,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 		return pi, toObjectErr(err)
 	}
 
-	partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader, dstOpts)
+	partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.PutObjReader, dstOpts)
 	if err != nil {
 		return pi, toObjectErr(err, dstBucket, dstObject)
 	}
@@ -271,7 +268,8 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 // an ongoing multipart transaction. Internally incoming data is
 // written to '.minio.sys/tmp' location and safely renamed to
 // '.minio.sys/multipart' for reach parts.
-func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, e error) {
+func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, e error) {
+	data := r.Reader
 	if err := checkPutObjectPartArgs(ctx, bucket, object, fs); err != nil {
 		return pi, toObjectErr(err, bucket)
 	}
@@ -321,11 +319,12 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	// PutObjectPart succeeds then there would be nothing to
 	// delete in which case we just ignore the error.
 	defer fsRemoveFile(ctx, tmpPartPath)
+	etag := r.MD5CurrentHexString()
 
-	etag := hex.EncodeToString(data.MD5Current())
 	if etag == "" {
 		etag = GenETag()
 	}
+
 	partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag, data.ActualSize()))
 
 	if err = fsRenameFile(ctx, tmpPartPath, partPath); err != nil {
@@ -478,7 +477,7 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 // md5sums of all the parts.
 //
 // Implements S3 compatible Complete multipart API.
-func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) {
+func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, e error) {
 
 	var actualSize int64
 

cmd/fs-v1-multipart_test.go

@@ -114,7 +114,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	sha256sum := ""
 
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{})
+	_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{})
 	if !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -145,7 +145,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 
 	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts); err != nil {
+	if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -175,13 +175,13 @@ func TestCompleteMultipartUpload(t *testing.T) {
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
+	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
 
-	if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts); err != nil {
+	if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 }
@@ -209,7 +209,7 @@ func TestAbortMultipartUpload(t *testing.T) {
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
+	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 	time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."

cmd/fs-v1.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -32,7 +31,6 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
@@ -189,14 +187,9 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
 		case <-doneCh:
 			return errWalkAbort
 		default:
-			var fi os.FileInfo
-			var err error
-			if hasSuffix(entry, slashSeparator) {
-				fi, err = fsStatDir(ctx, entry)
-			} else {
-				fi, err = fsStatFile(ctx, entry)
-			}
+			fi, err := os.Stat(entry)
 			if err != nil {
+				err = osErrToFSFileErr(err)
 				return err
 			}
 			atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
@@ -228,14 +221,9 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
 					}
 				}
 
-				var fi os.FileInfo
-				var err error
-				if hasSuffix(entry, slashSeparator) {
-					fi, err = fsStatDir(ctx, entry)
-				} else {
-					fi, err = fsStatFile(ctx, entry)
-				}
+				fi, err := os.Stat(entry)
 				if err != nil {
+					err = osErrToFSFileErr(err)
 					return err
 				}
 				usage = usage + uint64(fi.Size())
@@ -461,8 +449,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		// Return the new object info.
 		return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil
 	}
-
-	objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.Reader, srcInfo.UserDefined)
+	objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, srcInfo.UserDefined, dstOpts)
 	if err != nil {
 		return oi, toObjectErr(err, dstBucket, dstObject)
 	}
@@ -817,8 +804,8 @@ func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent strin
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
 // for future object operations.
-func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
-	if err := checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil {
+func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+	if err := checkPutObjectArgs(ctx, bucket, object, fs, r.Size()); err != nil {
 		return ObjectInfo{}, err
 	}
 	// Lock the object.
@@ -828,11 +815,13 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return objInfo, err
 	}
 	defer objectLock.Unlock()
-	return fs.putObject(ctx, bucket, object, data, metadata)
+	return fs.putObject(ctx, bucket, object, r, metadata, opts)
 }
 
 // putObject - wrapper for PutObject
-func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
+func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+	data := r.Reader
+
 	// No metadata is set, allocate a new one.
 	meta := make(map[string]string)
 	for k, v := range metadata {
@@ -923,8 +912,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 		fsRemoveFile(ctx, fsTmpObjPath)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
-
-	fsMeta.Meta["etag"] = hex.EncodeToString(data.MD5Current())
+	fsMeta.Meta["etag"] = r.MD5CurrentHexString()
 
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.

cmd/fs-v1_test.go

@@ -40,7 +40,7 @@ func TestFSParentDirIsObject(t *testing.T) {
 	}
 	objectContent := "12345"
 	objInfo, err := obj.PutObject(context.Background(), bucketName, objectName,
-		mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
+		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -124,7 +124,7 @@ func TestFSShutdown(t *testing.T) {
 
 		objectContent := "12345"
 		obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-		obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
+		obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
 		return fs, disk
 	}
 
@@ -203,7 +203,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a regular object.
-	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -212,7 +212,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a directory object.
-	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -220,11 +220,11 @@ func TestFSPutObject(t *testing.T) {
 		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backend corruption occurred")
 	}
@@ -239,7 +239,7 @@ func TestFSPutObject(t *testing.T) {
 		}
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backned corruption occurred")
 	}
@@ -267,7 +267,7 @@ func TestFSDeleteObject(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 
 	// Test with invalid bucket name
 	if err := fs.DeleteObject(context.Background(), "fo", objectName); !isSameType(err, BucketNameInvalid{}) {

cmd/gateway-main.go

@@ -41,7 +41,7 @@ func init() {
 var (
 	gatewayCmd = cli.Command{
 		Name:            "gateway",
-		Usage:           "Start object storage gateway.",
+		Usage:           "start object storage gateway",
 		Flags:           append(serverFlags, globalFlags...),
 		HideHelpCommand: true,
 	}
@@ -135,7 +135,8 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	handleCommonCmdArgs(ctx)
 
 	// Get port to listen on from gateway address
-	_, gatewayPort, pErr := net.SplitHostPort(gatewayAddr)
+	var pErr error
+	_, globalMinioPort, pErr = net.SplitHostPort(gatewayAddr)
 	if pErr != nil {
 		logger.FatalIf(pErr, "Unable to start gateway")
 	}
@@ -144,7 +145,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// to IPv6 address ie minio will start listening on IPv6 address whereas another
 	// (non-)minio process is listening on IPv4 of given port.
 	// To avoid this error situation we check for port availability.
-	logger.FatalIf(checkPortAvailability(gatewayPort), "Unable to start the gateway")
+	logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the gateway")
 
 	// Create certs path.
 	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
@@ -239,7 +240,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	loadLoggers()
 
 	// This is only to uniquely identify each gateway deployments.
-	logger.SetDeploymentID(os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID"))
+	globalDeploymentID = os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID")
 
 	var cacheConfig = globalServerConfig.GetCacheConfig()
 	if len(cacheConfig.Drives) > 0 {

cmd/gateway-unsupported.go

@@ -20,7 +20,6 @@ import (
 	"context"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -44,7 +43,8 @@ func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcOb
 }
 
 // PutObjectPart puts a part of object in bucket
-func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, err error) {
+func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
+	logger.LogIf(ctx, NotImplemented{})
 	return pi, NotImplemented{}
 }
 
@@ -60,7 +60,8 @@ func (a GatewayUnsupported) AbortMultipartUpload(ctx context.Context, bucket str
 }
 
 // CompleteMultipartUpload completes ongoing multipart upload and finalizes object
-func (a GatewayUnsupported) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) {
+func (a GatewayUnsupported) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
+	logger.LogIf(ctx, NotImplemented{})
 	return oi, NotImplemented{}
 }
 

cmd/gateway/azure/gateway-azure.go

@@ -38,7 +38,6 @@ import (
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/policy/condition"
 	sha256 "github.com/minio/sha256-simd"
@@ -737,7 +736,8 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 
 // PutObject - Create a new blob with the incoming data,
 // uses Azure equivalent CreateBlockBlobFromReader.
-func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	data := r.Reader
 	if data.Size() < azureBlockSize/10 {
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 		blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
@@ -938,7 +938,8 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st
 }
 
 // PutObjectPart - Use Azure equivalent PutBlockWithLength.
-func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (info minio.PartInfo, err error) {
+func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (info minio.PartInfo, err error) {
+	data := r.Reader
 	if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return info, err
 	}
@@ -1077,7 +1078,7 @@ func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object,
 }
 
 // CompleteMultipartUpload - Use Azure equivalent PutBlockList.
-func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	metadataObject := getAzureMetadataObjectName(object, uploadID)
 	if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return objInfo, err

cmd/gateway/b2/gateway-b2.go

@@ -92,7 +92,7 @@ EXAMPLES:
 `
 	minio.RegisterGatewayCommand(cli.Command{
 		Name:               b2Backend,
-		Usage:              "Backblaze B2.",
+		Usage:              "Backblaze B2",
 		Action:             b2GatewayMain,
 		CustomHelpTemplate: b2GatewayTemplate,
 		HideHelpCommand:    true,
@@ -534,7 +534,9 @@ func (nb *Reader) Read(p []byte) (int, error) {
 }
 
 // PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
-func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	data := r.Reader
+
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return objInfo, err
@@ -653,7 +655,8 @@ func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
 }
 
 // PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
-func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
+func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
+	data := r.Reader
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return pi, err
@@ -726,7 +729,7 @@ func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj
 }
 
 // CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
-func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) {
+func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, err error) {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return oi, err

cmd/gateway/gcs/gateway-gcs.go

@@ -40,7 +40,6 @@ import (
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/policy/condition"
 
@@ -148,7 +147,7 @@ EXAMPLES:
 
 	minio.RegisterGatewayCommand(cli.Command{
 		Name:               gcsBackend,
-		Usage:              "Google Cloud Storage.",
+		Usage:              "Google Cloud Storage",
 		Action:             gcsGatewayMain,
 		CustomHelpTemplate: gcsGatewayTemplate,
 		HideHelpCommand:    true,
@@ -207,11 +206,10 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 	gcs := &gcsGateway{
 		client:    client,
 		projectID: g.projectID,
-		ctx:       ctx,
 	}
 
 	// Start background process to cleanup old files in minio.sys.tmp
-	go gcs.CleanupGCSMinioSysTmp()
+	go gcs.CleanupGCSMinioSysTmp(ctx)
 	return gcs, nil
 }
 
@@ -350,7 +348,6 @@ type gcsGateway struct {
 	minio.GatewayUnsupported
 	client    *storage.Client
 	projectID string
-	ctx       context.Context
 }
 
 // Returns projectID from the GOOGLE_APPLICATION_CREDENTIALS file.
@@ -367,8 +364,8 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) {
 }
 
 // Cleanup old files in minio.sys.tmp of the given bucket.
-func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
-	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})
+func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket string) {
+	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})
 	for {
 		attrs, err := it.Next()
 		if err != nil {
@@ -381,7 +378,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
 		}
 		if time.Since(attrs.Updated) > gcsMultipartExpiry {
 			// Delete files older than 2 weeks.
-			err := l.client.Bucket(bucket).Object(attrs.Name).Delete(l.ctx)
+			err := l.client.Bucket(bucket).Object(attrs.Name).Delete(ctx)
 			if err != nil {
 				reqInfo := &logger.ReqInfo{BucketName: bucket, ObjectName: attrs.Name}
 				ctx := logger.SetReqInfo(context.Background(), reqInfo)
@@ -393,9 +390,9 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
 }
 
 // Cleanup old files in minio.sys.tmp of all buckets.
-func (l *gcsGateway) CleanupGCSMinioSysTmp() {
+func (l *gcsGateway) CleanupGCSMinioSysTmp(ctx context.Context) {
 	for {
-		it := l.client.Buckets(l.ctx, l.projectID)
+		it := l.client.Buckets(ctx, l.projectID)
 		for {
 			attrs, err := it.Next()
 			if err != nil {
@@ -405,7 +402,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmp() {
 				}
 				break
 			}
-			l.CleanupGCSMinioSysTmpBucket(attrs.Name)
+			l.CleanupGCSMinioSysTmpBucket(ctx, attrs.Name)
 		}
 		// Run the cleanup loop every 1 day.
 		time.Sleep(gcsCleanupInterval)
@@ -432,7 +429,7 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 		location = "us"
 	}
 
-	err := bkt.Create(l.ctx, l.projectID, &storage.BucketAttrs{
+	err := bkt.Create(ctx, l.projectID, &storage.BucketAttrs{
 		Location: location,
 	})
 	logger.LogIf(ctx, err)
@@ -441,7 +438,7 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 
 // GetBucketInfo - Get bucket metadata..
 func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.BucketInfo, error) {
-	attrs, err := l.client.Bucket(bucket).Attrs(l.ctx)
+	attrs, err := l.client.Bucket(bucket).Attrs(ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return minio.BucketInfo{}, gcsToObjectError(err, bucket)
@@ -455,7 +452,7 @@ func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.Bu
 
 // ListBuckets lists all buckets under your project-id on GCS.
 func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
-	it := l.client.Buckets(l.ctx, l.projectID)
+	it := l.client.Buckets(ctx, l.projectID)
 
 	// Iterate and capture all the buckets.
 	for {
@@ -480,7 +477,7 @@ func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
 
 // DeleteBucket delete a bucket on GCS.
 func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
-	itObject := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
+	itObject := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
 		Delimiter: "/",
 		Versions:  false,
 	})
@@ -510,7 +507,7 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
 	}
 	if gcsMinioPathFound {
 		// Remove minio.sys.tmp before deleting the bucket.
-		itObject = l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Versions: false, Prefix: minio.GatewayMinioSysTmp})
+		itObject = l.client.Bucket(bucket).Objects(ctx, &storage.Query{Versions: false, Prefix: minio.GatewayMinioSysTmp})
 		for {
 			objAttrs, err := itObject.Next()
 			if err == iterator.Done {
@@ -520,14 +517,14 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
 				logger.LogIf(ctx, err)
 				return gcsToObjectError(err)
 			}
-			err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(l.ctx)
+			err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(ctx)
 			if err != nil {
 				logger.LogIf(ctx, err)
 				return gcsToObjectError(err)
 			}
 		}
 	}
-	err := l.client.Bucket(bucket).Delete(l.ctx)
+	err := l.client.Bucket(bucket).Delete(ctx)
 	logger.LogIf(ctx, err)
 	return gcsToObjectError(err, bucket)
 }
@@ -562,7 +559,7 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
 		return minio.ListObjectsInfo{}, nil
 	}
 
-	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
+	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
 		Delimiter: delimiter,
 		Prefix:    prefix,
 		Versions:  false,
@@ -669,7 +666,7 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
 		return minio.ListObjectsV2Info{ContinuationToken: continuationToken}, nil
 	}
 
-	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
+	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
 		Delimiter: delimiter,
 		Prefix:    prefix,
 		Versions:  false,
@@ -770,7 +767,7 @@ func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string,
 func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
-	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
+	if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
 		logger.LogIf(ctx, err)
 		return gcsToObjectError(err, bucket)
 	}
@@ -782,7 +779,7 @@ func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, s
 	// Calling ReadCompressed with true accomplishes that.
 	object := l.client.Bucket(bucket).Object(key).ReadCompressed(true)
 
-	r, err := object.NewRangeReader(l.ctx, startOffset, length)
+	r, err := object.NewRangeReader(ctx, startOffset, length)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return gcsToObjectError(err, bucket, key)
@@ -865,12 +862,12 @@ func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAt
 func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
-	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
+	if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
 	}
 
-	attrs, err := l.client.Bucket(bucket).Object(object).Attrs(l.ctx)
+	attrs, err := l.client.Bucket(bucket).Object(object).Attrs(ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, bucket, object)
@@ -880,17 +877,19 @@ func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object st
 }
 
 // PutObject - Create a new object with the incoming data,
-func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
+func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
+	data := r.Reader
+
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
-	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
+	if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
 	}
 
 	object := l.client.Bucket(bucket).Object(key)
 
-	w := object.NewWriter(l.ctx)
+	w := object.NewWriter(ctx)
 	// Disable "chunked" uploading in GCS client if the size of the data to be uploaded is below
 	// the current chunk-size of the writer. This avoids an unnecessary memory allocation.
 	if data.Size() < int64(w.ChunkSize) {
@@ -908,7 +907,7 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d
 	// Close the object writer upon success.
 	w.Close()
 
-	attrs, err := object.Attrs(l.ctx)
+	attrs, err := object.Attrs(ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
@@ -927,7 +926,7 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
 	copier := dst.CopierFrom(src)
 	applyMetadataToGCSAttrs(srcInfo.UserDefined, &copier.ObjectAttrs)
 
-	attrs, err := copier.Run(l.ctx)
+	attrs, err := copier.Run(ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, destBucket, destObject)
@@ -938,7 +937,7 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
 
 // DeleteObject - Deletes a blob in bucket
 func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string) error {
-	err := l.client.Bucket(bucket).Object(object).Delete(l.ctx)
+	err := l.client.Bucket(bucket).Object(object).Delete(ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return gcsToObjectError(err, bucket, object)
@@ -955,7 +954,7 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key
 	// generate name for part zero
 	meta := gcsMultipartMetaName(uploadID)
 
-	w := l.client.Bucket(bucket).Object(meta).NewWriter(l.ctx)
+	w := l.client.Bucket(bucket).Object(meta).NewWriter(ctx)
 	defer w.Close()
 
 	applyMetadataToGCSAttrs(metadata, &w.ObjectAttrs)
@@ -1055,13 +1054,14 @@ func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, pr
 // Checks if minio.sys.tmp/multipart/v1/<upload-id>/gcs.json exists, returns
 // an object layer compatible error upon any error.
 func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key string, uploadID string) error {
-	_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx)
+	_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(ctx)
 	logger.LogIf(ctx, err)
 	return gcsToObjectError(err, bucket, key, uploadID)
 }
 
 // PutObjectPart puts a part of object in bucket
-func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader, opts minio.ObjectOptions) (minio.PartInfo, error) {
+func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, r *minio.PutObjReader, opts minio.ObjectOptions) (minio.PartInfo, error) {
+	data := r.Reader
 	if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil {
 		return minio.PartInfo{}, err
 	}
@@ -1071,7 +1071,7 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin
 		etag = minio.GenETag()
 	}
 	object := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, partNumber, etag))
-	w := object.NewWriter(l.ctx)
+	w := object.NewWriter(ctx)
 	// Disable "chunked" uploading in GCS client. If enabled, it can cause a corner case
 	// where it tries to upload 0 bytes in the last chunk and get error from server.
 	w.ChunkSize = 0
@@ -1184,7 +1184,7 @@ func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, up
 	prefix := fmt.Sprintf("%s/%s/", gcsMinioMultipartPathV1, uploadID)
 
 	// iterate through all parts and delete them
-	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Prefix: prefix, Versions: false})
+	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: prefix, Versions: false})
 
 	for {
 		attrs, err := it.Next()
@@ -1198,7 +1198,7 @@ func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, up
 
 		object := l.client.Bucket(bucket).Object(attrs.Name)
 		// Ignore the error as parallel AbortMultipartUpload might have deleted it.
-		object.Delete(l.ctx)
+		object.Delete(ctx)
 	}
 
 	return nil
@@ -1216,17 +1216,17 @@ func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, ke
 // Note that there is a limit (currently 32) to the number of components that can
 // be composed in a single operation. There is a per-project rate limit (currently 200)
 // to the number of source objects you can compose per second.
-func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, uploadedParts []minio.CompletePart) (minio.ObjectInfo, error) {
+func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	meta := gcsMultipartMetaName(uploadID)
 	object := l.client.Bucket(bucket).Object(meta)
 
-	partZeroAttrs, err := object.Attrs(l.ctx)
+	partZeroAttrs, err := object.Attrs(ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key, uploadID)
 	}
 
-	r, err := object.NewReader(l.ctx)
+	r, err := object.NewReader(ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
@@ -1257,7 +1257,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
 	for i, uploadedPart := range uploadedParts {
 		parts = append(parts, l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID,
 			uploadedPart.PartNumber, uploadedPart.ETag)))
-		partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(l.ctx)
+		partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(ctx)
 		if pErr != nil {
 			logger.LogIf(ctx, pErr)
 			return minio.ObjectInfo{}, gcsToObjectError(pErr, bucket, key, uploadID)
@@ -1303,7 +1303,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
 			composer.ContentType = partZeroAttrs.ContentType
 			composer.Metadata = partZeroAttrs.Metadata
 
-			if _, err = composer.Run(l.ctx); err != nil {
+			if _, err = composer.Run(ctx); err != nil {
 				logger.LogIf(ctx, err)
 				return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
 			}
@@ -1320,7 +1320,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
 	composer.ContentDisposition = partZeroAttrs.ContentDisposition
 	composer.ContentLanguage = partZeroAttrs.ContentLanguage
 	composer.Metadata = partZeroAttrs.Metadata
-	attrs, err := composer.Run(l.ctx)
+	attrs, err := composer.Run(ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
@@ -1360,7 +1360,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
 
 	acl := l.client.Bucket(bucket).ACL()
 	if policies[0].Policy == miniogopolicy.BucketPolicyNone {
-		if err := acl.Delete(l.ctx, storage.AllUsers); err != nil {
+		if err := acl.Delete(ctx, storage.AllUsers); err != nil {
 			logger.LogIf(ctx, err)
 			return gcsToObjectError(err, bucket)
 		}
@@ -1378,7 +1378,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
 		return minio.NotImplemented{}
 	}
 
-	if err := acl.Set(l.ctx, storage.AllUsers, role); err != nil {
+	if err := acl.Set(ctx, storage.AllUsers, role); err != nil {
 		logger.LogIf(ctx, err)
 		return gcsToObjectError(err, bucket)
 	}
@@ -1388,7 +1388,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
 
 // GetBucketPolicy - Get policy on bucket
 func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
+	rules, err := l.client.Bucket(bucket).ACL().List(ctx)
 	if err != nil {
 		return nil, gcsToObjectError(err, bucket)
 	}
@@ -1447,7 +1447,7 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic
 // DeleteBucketPolicy - Delete all policies on bucket
 func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error {
 	// This only removes the storage.AllUsers policies
-	if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil {
+	if err := l.client.Bucket(bucket).ACL().Delete(ctx, storage.AllUsers); err != nil {
 		return gcsToObjectError(err, bucket)
 	}
 

cmd/gateway/manta/gateway-manta.go

@@ -35,7 +35,6 @@ import (
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/hash"
 )
 
 // stor is a namespace within manta where you store any documents that are deemed as private
@@ -109,7 +108,7 @@ EXAMPLES:
 
 	minio.RegisterGatewayCommand(cli.Command{
 		Name:               mantaBackend,
-		Usage:              "Manta Object Storage.",
+		Usage:              "Manta Object Storage",
 		Action:             mantaGatewayMain,
 		CustomHelpTemplate: mantaGatewayTemplate,
 		HideHelpCommand:    true,
@@ -608,7 +607,8 @@ func (d dummySeeker) Seek(offset int64, whence int) (int64, error) {
 // CreateBlockBlobFromReader.
 //
 // https://apidocs.joyent.com/manta/api.html#PutObject
-func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	data := r.Reader
 	if err = t.client.Objects().Put(ctx, &storage.PutObjectInput{
 		ContentLength: uint64(data.Size()),
 		ObjectPath:    path.Join(mantaRoot, bucket, object),

cmd/gateway/nas/gateway-nas.go

@@ -74,7 +74,7 @@ EXAMPLES:
 
 	minio.RegisterGatewayCommand(cli.Command{
 		Name:               nasBackend,
-		Usage:              "Network-attached storage (NAS).",
+		Usage:              "Network-attached storage (NAS)",
 		Action:             nasGatewayMain,
 		CustomHelpTemplate: nasGatewayTemplate,
 		HideHelpCommand:    true,

cmd/gateway/oss/gateway-oss.go

@@ -98,7 +98,7 @@ EXAMPLES:
 
 	minio.RegisterGatewayCommand(cli.Command{
 		Name:               "oss",
-		Usage:              "Alibaba Cloud (Aliyun) Object Storage Service (OSS).",
+		Usage:              "Alibaba Cloud (Aliyun) Object Storage Service (OSS)",
 		Action:             ossGatewayMain,
 		CustomHelpTemplate: ossGatewayTemplate,
 		HideHelpCommand:    true,
@@ -655,7 +655,9 @@ func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string
 }
 
 // PutObject creates a new object with the incoming data.
-func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	data := r.Reader
+
 	return ossPutObject(ctx, l.Client, bucket, object, data, metadata)
 }
 
@@ -773,7 +775,8 @@ func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
 }
 
 // PutObjectPart puts a part of object in bucket.
-func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
+func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
+	data := r.Reader
 	bkt, err := l.Client.Bucket(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -914,7 +917,7 @@ func (l *ossObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 }
 
 // CompleteMultipartUpload completes ongoing multipart upload and finalizes object.
-func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) {
+func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, err error) {
 	client := l.Client
 	bkt, err := client.Bucket(bucket)
 	if err != nil {

cmd/gateway/s3/gateway-s3.go

@@ -31,7 +31,6 @@ import (
 	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
 
 	minio "github.com/minio/minio/cmd"
@@ -112,7 +111,7 @@ EXAMPLES:
 
 	minio.RegisterGatewayCommand(cli.Command{
 		Name:               s3Backend,
-		Usage:              "Amazon Simple Storage Service (S3).",
+		Usage:              "Amazon Simple Storage Service (S3)",
 		Action:             s3GatewayMain,
 		CustomHelpTemplate: s3GatewayTemplate,
 		HideHelpCommand:    true,
@@ -406,7 +405,8 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 }
 
 // PutObject creates a new object with the incoming data,
-func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	data := r.Reader
 	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata), opts.ServerSideEncryption)
 	if err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
@@ -464,7 +464,8 @@ func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
 }
 
 // PutObjectPart puts a part of object in bucket
-func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, e error) {
+func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, e error) {
+	data := r.Reader
 	info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), opts.ServerSideEncryption)
 	if err != nil {
 		return pi, minio.ErrorRespToObjectError(err, bucket, object)
@@ -508,13 +509,13 @@ func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj
 }
 
 // CompleteMultipartUpload completes ongoing multipart upload and finalizes object
-func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, e error) {
-	err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts))
+func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, e error) {
+	etag, err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts))
 	if err != nil {
 		return oi, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
-	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
+	return minio.ObjectInfo{Bucket: bucket, Name: object, ETag: etag}, nil
 }
 
 // SetBucketPolicy sets policy on bucket

cmd/gateway/sia/gateway-sia.go

@@ -38,7 +38,6 @@ import (
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/sha256-simd"
 )
 
@@ -98,7 +97,7 @@ EXAMPLES:
 
 	minio.RegisterGatewayCommand(cli.Command{
 		Name:               siaBackend,
-		Usage:              "Sia Decentralized Cloud.",
+		Usage:              "Sia Decentralized Cloud",
 		Action:             siaGatewayMain,
 		CustomHelpTemplate: siaGatewayTemplate,
 		HideHelpCommand:    true,
@@ -554,7 +553,8 @@ func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object st
 }
 
 // PutObject creates a new object with the incoming data,
-func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	data := r.Reader
 	srcFile := path.Join(s.TempDir, minio.MustGetUUID())
 	writer, err := os.Create(srcFile)
 	if err != nil {

cmd/generic-handlers.go

@@ -91,7 +91,7 @@ func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
 // of the user-defined metadata to 2 KB.
 func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if isHTTPHeaderSizeTooLarge(r.Header) {
-		writeErrorResponse(w, ErrMetadataTooLarge, r.URL)
+		writeErrorResponse(w, ErrMetadataTooLarge, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	h.Handler.ServeHTTP(w, r)
@@ -134,7 +134,7 @@ func filterReservedMetadata(h http.Handler) http.Handler {
 // would be treated as metadata.
 func (h reservedMetadataHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if containsReservedMetadata(r.Header) {
-		writeErrorResponse(w, ErrUnsupportedMetadata, r.URL)
+		writeErrorResponse(w, ErrUnsupportedMetadata, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	h.Handler.ServeHTTP(w, r)
@@ -300,7 +300,7 @@ func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 		// buckets
 		bucketName, _ := urlPath2BucketObjectName(r.URL.Path)
 		if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
-			writeErrorResponse(w, ErrAllAccessDisabled, r.URL)
+			writeErrorResponse(w, ErrAllAccessDisabled, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -363,14 +363,14 @@ func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			// All our internal APIs are sensitive towards Date
 			// header, for all requests where Date header is not
 			// present we will reject such clients.
-			writeErrorResponse(w, apiErr, r.URL)
+			writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
 			return
 		}
 		// Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past
 		// or in the future, reject request otherwise.
 		curTime := UTCNow()
 		if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
-			writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL)
+			writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -485,20 +485,20 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// If bucketName is present and not objectName check for bucket level resource queries.
 	if bucketName != "" && objectName == "" {
 		if ignoreNotImplementedBucketResources(r) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL)
+			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 	// If bucketName and objectName are present check for its resource queries.
 	if bucketName != "" && objectName != "" {
 		if ignoreNotImplementedObjectResources(r) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL)
+			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 	// A put method on path "/" doesn't make sense, ignore it.
 	if r.Method == http.MethodPut && r.URL.Path == "/" {
-		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -601,14 +601,14 @@ func hasBadPathComponent(path string) bool {
 func (h pathValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Check for bad components in URL path.
 	if hasBadPathComponent(r.URL.Path) {
-		writeErrorResponse(w, ErrInvalidResourceName, r.URL)
+		writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// Check for bad components in URL query values.
 	for _, vv := range r.URL.Query() {
 		for _, v := range vv {
 			if hasBadPathComponent(v) {
-				writeErrorResponse(w, ErrInvalidResourceName, r.URL)
+				writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
@@ -625,7 +625,9 @@ type bucketForwardingHandler struct {
 }
 
 func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if globalDNSConfig == nil || globalDomainName == "" || guessIsBrowserReq(r) || guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || guessIsRPCReq(r) {
+	if globalDNSConfig == nil || globalDomainName == "" ||
+		guessIsBrowserReq(r) || guessIsHealthCheckReq(r) ||
+		guessIsMetricsReq(r) || guessIsRPCReq(r) || isAdminReq(r) {
 		f.handler.ServeHTTP(w, r)
 		return
 	}
@@ -653,9 +655,9 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 	sr, err := globalDNSConfig.Get(bucket)
 	if err != nil {
 		if err == dns.ErrNoEntriesFound {
-			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+			writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
 		} else {
-			writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL)
+			writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
 		}
 		return
 	}
@@ -716,31 +718,32 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// potential pileup on the server.
 	if err := l.Wait(ctx); err != nil {
 		// Send an S3 compatible error, SlowDown.
-		writeErrorResponse(w, ErrSlowDown, r.URL)
+		writeErrorResponse(w, ErrSlowDown, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	l.handler.ServeHTTP(w, r)
 }
 
-// requestIDHeaderHandler sets x-amz-request-id header.
-// Previously, this value was set right before a response
-// was sent to the client.So, logger and Error response XML
-// were not using this value.
-// This is set here so that this header can be logged as
-// part of the log entry and Error response XML.
-type requestIDHeaderHandler struct {
+// customHeaderHandler sets x-amz-request-id, x-minio-deployment-id header.
+// Previously, this value was set right before a response was sent to
+// the client. So, logger and Error response XML were not using this
+// value. This is set here so that this header can be logged as
+// part of the log entry, Error response XML and auditing.
+type customHeaderHandler struct {
 	handler http.Handler
 }
 
-func addrequestIDHeader(h http.Handler) http.Handler {
-	return requestIDHeaderHandler{handler: h}
+func addCustomHeaders(h http.Handler) http.Handler {
+	return customHeaderHandler{handler: h}
 }
 
-func (s requestIDHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// Set unique request ID for each response.
+func (s customHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Set custom headers such as x-amz-request-id and x-minio-deployment-id
+	// for each request.
 	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
-	s.handler.ServeHTTP(w, r)
+	w.Header().Set(responseDeploymentIDKey, globalDeploymentID)
+	s.handler.ServeHTTP(logger.NewResponseWriter(w), r)
 }
 
 type securityHeaderHandler struct {
@@ -767,7 +770,7 @@ type criticalErrorHandler struct{ handler http.Handler }
 func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer func() {
 		if err := recover(); err == logger.ErrCritical { // handle
-			writeErrorResponse(w, ErrInternalError, r.URL)
+			writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
 		} else if err != nil {
 			panic(err) // forward other panic calls
 		}
@@ -786,7 +789,7 @@ func (h sseTLSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if r.Method == http.MethodHead {
 			writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
 		} else {
-			writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL)
+			writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL, guessIsBrowserReq(r))
 		}
 		return
 	}

cmd/globals.go

@@ -255,6 +255,9 @@ var (
 	// OPA policy system.
 	globalPolicyOPA *iampolicy.Opa
 
+	// Deployment ID - unique per deployment
+	globalDeploymentID string
+
 	// Add new variable global values here.
 )
 

cmd/handler-utils.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
 	httptracer "github.com/minio/minio/pkg/handlers"
 )
@@ -176,13 +177,13 @@ func getRedirectPostRawQuery(objInfo ObjectInfo) string {
 	return redirectValues.Encode()
 }
 
-// Returns access key in the request Authorization header.
-func getReqAccessKey(r *http.Request, region string) (accessKey string) {
-	cred, _, _ := getReqAccessKeyV4(r, region)
+// Returns access credentials in the request Authorization header.
+func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
+	cred, _, _ = getReqAccessKeyV4(r, region)
 	if cred.AccessKey == "" {
 		cred, _, _ = getReqAccessKeyV2(r)
 	}
-	return cred.AccessKey
+	return cred
 }
 
 // Extract request params to be sent with event notifiation.
@@ -192,10 +193,11 @@ func extractReqParams(r *http.Request) map[string]string {
 	}
 
 	region := globalServerConfig.GetRegion()
+	cred := getReqAccessCred(r, region)
 	// Success.
 	return map[string]string{
 		"region":          region,
-		"accessKey":       getReqAccessKey(r, region),
+		"accessKey":       cred.AccessKey,
 		"sourceIPAddress": handlers.GetSourceIP(r),
 		// Add more fields here.
 	}
@@ -351,6 +353,6 @@ func getResource(path string, host string, domain string) (string, error) {
 
 // If none of the http routes match respond with MethodNotAllowed
 func notFoundHandler(w http.ResponseWriter, r *http.Request) {
-	writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+	writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
 	return
 }

cmd/http/listen_others.go

@@ -1,4 +1,4 @@
-// +build windows plan9
+// +build windows plan9 solaris
 
 /*
  * Minio Cloud Storage, (C) 2018 Minio, Inc.

cmd/logger/audit-logger.go

@@ -1,97 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package logger
-
-import (
-	"context"
-	"net/http"
-	"strings"
-	"time"
-)
-
-// Represents the current version of audit log structure.
-const auditLogVersion = "1"
-
-// AuditEntry - audit entry logs.
-type AuditEntry struct {
-	Version      string            `json:"version"`
-	DeploymentID string            `json:"deploymentid,omitempty"`
-	Time         string            `json:"time"`
-	API          *api              `json:"api,omitempty"`
-	RemoteHost   string            `json:"remotehost,omitempty"`
-	RequestID    string            `json:"requestID,omitempty"`
-	UserAgent    string            `json:"userAgent,omitempty"`
-	ReqQuery     map[string]string `json:"requestQuery,omitempty"`
-	ReqHeader    map[string]string `json:"requestHeader,omitempty"`
-	RespHeader   map[string]string `json:"responseHeader,omitempty"`
-}
-
-// AuditTargets is the list of enabled audit loggers
-var AuditTargets = []LoggingTarget{}
-
-// AddAuditTarget adds a new audit logger target to the
-// list of enabled loggers
-func AddAuditTarget(t LoggingTarget) {
-	AuditTargets = append(AuditTargets, t)
-}
-
-// AuditLog - logs audit logs to all targets.
-func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	if Disable {
-		return
-	}
-
-	req := GetReqInfo(ctx)
-	if req == nil {
-		req = &ReqInfo{API: "SYSTEM"}
-	}
-
-	reqQuery := make(map[string]string)
-	for k, v := range r.URL.Query() {
-		reqQuery[k] = strings.Join(v, ",")
-	}
-	reqHeader := make(map[string]string)
-	for k, v := range r.Header {
-		reqHeader[k] = strings.Join(v, ",")
-	}
-	respHeader := make(map[string]string)
-	for k, v := range w.Header() {
-		respHeader[k] = strings.Join(v, ",")
-	}
-
-	// Send audit logs only to http targets.
-	for _, t := range AuditTargets {
-		t.send(AuditEntry{
-			Version:      auditLogVersion,
-			DeploymentID: deploymentID,
-			RemoteHost:   req.RemoteHost,
-			RequestID:    req.RequestID,
-			UserAgent:    req.UserAgent,
-			Time:         time.Now().UTC().Format(time.RFC3339Nano),
-			API: &api{
-				Name: req.API,
-				Args: &args{
-					Bucket: req.BucketName,
-					Object: req.ObjectName,
-				},
-			},
-			ReqQuery:   reqQuery,
-			ReqHeader:  reqHeader,
-			RespHeader: respHeader,
-		})
-	}
-}

cmd/logger/audit.go

@@ -0,0 +1,68 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"net/http"
+
+	"github.com/minio/minio/cmd/logger/message/audit"
+)
+
+// ResponseWriter - is a wrapper to trap the http response status code.
+type ResponseWriter struct {
+	http.ResponseWriter
+	statusCode int
+}
+
+// NewResponseWriter - returns a wrapped response writer to trap
+// http status codes for auditiing purposes.
+func NewResponseWriter(w http.ResponseWriter) *ResponseWriter {
+	return &ResponseWriter{w, http.StatusOK}
+}
+
+// WriteHeader - writes http status code
+func (lrw *ResponseWriter) WriteHeader(code int) {
+	lrw.statusCode = code
+	lrw.ResponseWriter.WriteHeader(code)
+}
+
+// Flush - Calls the underlying Flush.
+func (lrw *ResponseWriter) Flush() {
+	lrw.ResponseWriter.(http.Flusher).Flush()
+}
+
+// AuditTargets is the list of enabled audit loggers
+var AuditTargets = []Target{}
+
+// AddAuditTarget adds a new audit logger target to the
+// list of enabled loggers
+func AddAuditTarget(t Target) {
+	AuditTargets = append(AuditTargets, t)
+}
+
+// AuditLog - logs audit logs to all audit targets.
+func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[string]interface{}) {
+	var statusCode int
+	lrw, ok := w.(*ResponseWriter)
+	if ok {
+		statusCode = lrw.statusCode
+	}
+	// Send audit logs only to http targets.
+	for _, t := range AuditTargets {
+		_ = t.Send(audit.ToEntry(w, r, api, statusCode, reqClaims))
+	}
+}

cmd/logger/console.go

@@ -19,87 +19,168 @@ package logger
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"strings"
 	"time"
+
+	c "github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/cmd/logger/message/log"
 )
 
-// ConsoleTarget implements loggerTarget to send log
-// in plain or json format to the standard output.
-type ConsoleTarget struct{}
+// Console interface describes the methods that need to be implemented to satisfy the interface requirements.
+type Console interface {
+	json(msg string, args ...interface{})
+	quiet(msg string, args ...interface{})
+	pretty(msg string, args ...interface{})
+}
 
-func (c *ConsoleTarget) send(e interface{}) error {
-	entry, ok := e.(logEntry)
-	if !ok {
-		return fmt.Errorf("Uexpected log entry structure %#v", e)
+func consoleLog(console Console, msg string, args ...interface{}) {
+	switch {
+	case jsonFlag:
+		// Strip escape control characters from json message
+		msg = ansiRE.ReplaceAllLiteralString(msg, "")
+		console.json(msg, args...)
+	case quietFlag:
+		console.quiet(msg, args...)
+	default:
+		console.pretty(msg, args...)
 	}
-	if jsonFlag {
-		logJSON, err := json.Marshal(&entry)
-		if err != nil {
-			return err
+}
+
+// Fatal prints only fatal error message with no stack trace
+// it will be called for input validation failures
+func Fatal(err error, msg string, data ...interface{}) {
+	fatal(err, msg, data...)
+}
+
+func fatal(err error, msg string, data ...interface{}) {
+	var errMsg string
+	if msg != "" {
+		errMsg = errorFmtFunc(fmt.Sprintf(msg, data...), err, jsonFlag)
+	} else {
+		errMsg = err.Error()
+	}
+	consoleLog(fatalMessage, errMsg)
+}
+
+var fatalMessage fatalMsg
+
+type fatalMsg struct {
+}
+
+func (f fatalMsg) json(msg string, args ...interface{}) {
+	logJSON, err := json.Marshal(&log.Entry{
+		Level: FatalLvl.String(),
+		Time:  time.Now().UTC().Format(time.RFC3339Nano),
+		Trace: &log.Trace{Message: fmt.Sprintf(msg, args...), Source: []string{getSource(6)}},
+	})
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println(string(logJSON))
+
+	os.Exit(1)
+
+}
+
+func (f fatalMsg) quiet(msg string, args ...interface{}) {
+	f.pretty(msg, args...)
+}
+
+var (
+	logTag      = "ERROR"
+	logBanner   = ColorBgRed(ColorFgWhite(ColorBold(logTag))) + " "
+	emptyBanner = ColorBgRed(strings.Repeat(" ", len(logTag))) + " "
+	bannerWidth = len(logTag) + 1
+)
+
+func (f fatalMsg) pretty(msg string, args ...interface{}) {
+	// Build the passed error message
+	errMsg := fmt.Sprintf(msg, args...)
+
+	tagPrinted := false
+
+	// Print the error message: the following code takes care
+	// of splitting error text and always pretty printing the
+	// red banner along with the error message. Since the error
+	// message itself contains some colored text, we needed
+	// to use some ANSI control escapes to cursor color state
+	// and freely move in the screen.
+	for _, line := range strings.Split(errMsg, "\n") {
+		if len(line) == 0 {
+			// No more text to print, just quit.
+			break
 		}
-		fmt.Println(string(logJSON))
-		return nil
-	}
 
-	trace := make([]string, len(entry.Trace.Source))
-
-	// Add a sequence number and formatting for each stack trace
-	// No formatting is required for the first entry
-	for i, element := range entry.Trace.Source {
-		trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
-	}
-
-	tagString := ""
-	for key, value := range entry.Trace.Variables {
-		if value != "" {
-			if tagString != "" {
-				tagString += ", "
+		for {
+			// Save the attributes of the current cursor helps
+			// us save the text color of the passed error message
+			ansiSaveAttributes()
+			// Print banner with or without the log tag
+			if !tagPrinted {
+				fmt.Print(logBanner)
+				tagPrinted = true
+			} else {
+				fmt.Print(emptyBanner)
 			}
-			tagString += key + "=" + value
+			// Restore the text color of the error message
+			ansiRestoreAttributes()
+			ansiMoveRight(bannerWidth)
+			// Continue  error message printing
+			fmt.Println(line)
+			break
 		}
 	}
 
-	apiString := "API: " + entry.API.Name + "("
-	if entry.API.Args != nil && entry.API.Args.Bucket != "" {
-		apiString = apiString + "bucket=" + entry.API.Args.Bucket
-	}
-	if entry.API.Args != nil && entry.API.Args.Object != "" {
-		apiString = apiString + ", object=" + entry.API.Args.Object
-	}
-	apiString += ")"
-	timeString := "Time: " + time.Now().Format(loggerTimeFormat)
-
-	var requestID string
-	if entry.RequestID != "" {
-		requestID = "\nRequestID: " + entry.RequestID
-	}
-
-	var remoteHost string
-	if entry.RemoteHost != "" {
-		remoteHost = "\nRemoteHost: " + entry.RemoteHost
-	}
-
-	var userAgent string
-	if entry.UserAgent != "" {
-		userAgent = "\nUserAgent: " + entry.UserAgent
-	}
-
-	if len(entry.Trace.Variables) > 0 {
-		tagString = "\n       " + tagString
-	}
-
-	var msg = colorFgRed(colorBold(entry.Trace.Message))
-	var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
-		apiString, timeString, requestID, remoteHost, userAgent,
-		msg, tagString, strings.Join(trace, "\n"))
-
-	fmt.Println(output)
-	return nil
+	// Exit because this is a fatal error message
+	os.Exit(1)
 }
 
-// NewConsole initializes a new logger target
-// which prints log directly in the standard
-// output.
-func NewConsole() LoggingTarget {
-	return &ConsoleTarget{}
+type infoMsg struct{}
+
+var info infoMsg
+
+func (i infoMsg) json(msg string, args ...interface{}) {
+	logJSON, err := json.Marshal(&log.Entry{
+		Level:   InformationLvl.String(),
+		Message: fmt.Sprintf(msg, args...),
+		Time:    time.Now().UTC().Format(time.RFC3339Nano),
+	})
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println(string(logJSON))
+}
+
+func (i infoMsg) quiet(msg string, args ...interface{}) {
+	i.pretty(msg, args...)
+}
+
+func (i infoMsg) pretty(msg string, args ...interface{}) {
+	c.Printf(msg, args...)
+}
+
+// Info :
+func Info(msg string, data ...interface{}) {
+	consoleLog(info, msg+"\n", data...)
+}
+
+var startupMessage startUpMsg
+
+type startUpMsg struct {
+}
+
+func (s startUpMsg) json(msg string, args ...interface{}) {
+}
+
+func (s startUpMsg) quiet(msg string, args ...interface{}) {
+}
+
+func (s startUpMsg) pretty(msg string, args ...interface{}) {
+	c.Printf(msg, args...)
+}
+
+// StartupMessage :
+func StartupMessage(msg string, data ...interface{}) {
+	consoleLog(startupMessage, msg+"\n", data...)
 }

cmd/logger/logger.go

@@ -18,23 +18,20 @@ package logger
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"go/build"
-	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
 
-	c "github.com/minio/mc/pkg/console"
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/logger/message/log"
 )
 
 // Disable disables all logging, false by default. (used for "go test")
 var Disable = false
 
-var trimStrings []string
-
 // Level type
 type Level int8
 
@@ -45,7 +42,10 @@ const (
 	FatalLvl
 )
 
-const loggerTimeFormat string = "15:04:05 MST 01/02/2006"
+var trimStrings []string
+
+// TimeFormat - logging time format.
+const TimeFormat string = "15:04:05 MST 01/02/2006"
 
 // List of error strings to be ignored by LogIf
 const (
@@ -91,79 +91,33 @@ func (level Level) String() string {
 	return lvlStr
 }
 
-// Console interface describes the methods that needs to be implemented to satisfy the interface requirements.
-type Console interface {
-	json(msg string, args ...interface{})
-	quiet(msg string, args ...interface{})
-	pretty(msg string, args ...interface{})
-}
-
-func consoleLog(console Console, msg string, args ...interface{}) {
-	switch {
-	case jsonFlag:
-		// Strip escape control characters from json message
-		msg = ansiRE.ReplaceAllLiteralString(msg, "")
-		console.json(msg, args...)
-	case quiet:
-		console.quiet(msg, args...)
-	default:
-		console.pretty(msg, args...)
-	}
-}
-
-type traceEntry struct {
-	Message   string            `json:"message,omitempty"`
-	Source    []string          `json:"source,omitempty"`
-	Variables map[string]string `json:"variables,omitempty"`
-}
-
-type args struct {
-	Bucket   string            `json:"bucket,omitempty"`
-	Object   string            `json:"object,omitempty"`
-	Metadata map[string]string `json:"metadata,omitempty"`
-}
-
-type api struct {
-	Name string `json:"name,omitempty"`
-	Args *args  `json:"args,omitempty"`
-}
-
-type logEntry struct {
-	DeploymentID string      `json:"deploymentid,omitempty"`
-	Level        string      `json:"level"`
-	Time         string      `json:"time"`
-	API          *api        `json:"api,omitempty"`
-	RemoteHost   string      `json:"remotehost,omitempty"`
-	RequestID    string      `json:"requestID,omitempty"`
-	UserAgent    string      `json:"userAgent,omitempty"`
-	Message      string      `json:"message,omitempty"`
-	Trace        *traceEntry `json:"error,omitempty"`
-}
-
-// quiet: Hide startup messages if enabled
+// quietFlag: Hide startup messages if enabled
 // jsonFlag: Display in JSON format, if enabled
 var (
-	quiet, jsonFlag bool
+	quietFlag, jsonFlag bool
 	// Custom function to format error
 	errorFmtFunc func(string, error, bool) string
-
-	deploymentID string
 )
 
-// SetDeploymentID - Used to set the deployment ID, in XL and FS mode
-func SetDeploymentID(id string) {
-	deploymentID = id
-}
-
 // EnableQuiet - turns quiet option on.
 func EnableQuiet() {
-	quiet = true
+	quietFlag = true
 }
 
 // EnableJSON - outputs logs in json format.
 func EnableJSON() {
 	jsonFlag = true
-	quiet = true
+	quietFlag = true
+}
+
+// IsJSON - returns true if jsonFlag is true
+func IsJSON() bool {
+	return jsonFlag
+}
+
+// IsQuiet - returns true if quietFlag is true
+func IsQuiet() bool {
+	return quietFlag
 }
 
 // RegisterUIError registers the specified rendering function. This latter
@@ -172,6 +126,17 @@ func RegisterUIError(f func(string, error, bool) string) {
 	errorFmtFunc = f
 }
 
+// Remove any duplicates and return unique entries.
+func uniqueEntries(paths []string) []string {
+	m := make(set.StringSet)
+	for _, p := range paths {
+		if !m.Contains(p) {
+			m.Add(p)
+		}
+	}
+	return m.ToSlice()
+}
+
 // Init sets the trimStrings to possible GOPATHs
 // and GOROOT directories. Also append github.com/minio/minio
 // This is done to clean up the filename, when stack trace is
@@ -326,20 +291,30 @@ func logIf(ctx context.Context, err error) {
 	// Get the cause for the Error
 	message := err.Error()
 
-	entry := logEntry{
-		DeploymentID: deploymentID,
+	entry := log.Entry{
+		DeploymentID: req.DeploymentID,
 		Level:        ErrorLvl.String(),
 		RemoteHost:   req.RemoteHost,
 		RequestID:    req.RequestID,
 		UserAgent:    req.UserAgent,
 		Time:         time.Now().UTC().Format(time.RFC3339Nano),
-		API:          &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
-		Trace:        &traceEntry{Message: message, Source: trace, Variables: tags},
+		API: &log.API{
+			Name: API,
+			Args: &log.Args{
+				Bucket: req.BucketName,
+				Object: req.ObjectName,
+			},
+		},
+		Trace: &log.Trace{
+			Message:   message,
+			Source:    trace,
+			Variables: tags,
+		},
 	}
 
 	// Iterate over all logger targets to send the log entry
 	for _, t := range Targets {
-		t.send(entry)
+		t.Send(entry)
 	}
 }
 
@@ -362,141 +337,3 @@ func FatalIf(err error, msg string, data ...interface{}) {
 	}
 	fatal(err, msg, data...)
 }
-
-// Fatal prints only fatal error message without no stack trace
-// it will be called for input validation failures
-func Fatal(err error, msg string, data ...interface{}) {
-	fatal(err, msg, data...)
-}
-
-func fatal(err error, msg string, data ...interface{}) {
-	var errMsg string
-	if msg != "" {
-		errMsg = errorFmtFunc(fmt.Sprintf(msg, data...), err, jsonFlag)
-	} else {
-		errMsg = err.Error()
-	}
-	consoleLog(fatalMessage, errMsg)
-}
-
-var fatalMessage fatalMsg
-
-type fatalMsg struct {
-}
-
-func (f fatalMsg) json(msg string, args ...interface{}) {
-	logJSON, err := json.Marshal(&logEntry{
-		Level: FatalLvl.String(),
-		Time:  time.Now().UTC().Format(time.RFC3339Nano),
-		Trace: &traceEntry{Message: fmt.Sprintf(msg, args...), Source: []string{getSource(6)}},
-	})
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println(string(logJSON))
-
-	os.Exit(1)
-
-}
-
-func (f fatalMsg) quiet(msg string, args ...interface{}) {
-	f.pretty(msg, args...)
-}
-
-var (
-	logTag      = "ERROR"
-	logBanner   = colorBgRed(colorFgWhite(colorBold(logTag))) + " "
-	emptyBanner = colorBgRed(strings.Repeat(" ", len(logTag))) + " "
-	bannerWidth = len(logTag) + 1
-)
-
-func (f fatalMsg) pretty(msg string, args ...interface{}) {
-	// Build the passed error message
-	errMsg := fmt.Sprintf(msg, args...)
-
-	tagPrinted := false
-
-	// Print the error message: the following code takes care
-	// of splitting error text and always pretty printing the
-	// red banner along with the error message. Since the error
-	// message itself contains some colored text, we needed
-	// to use some ANSI control escapes to cursor color state
-	// and freely move in the screen.
-	for _, line := range strings.Split(errMsg, "\n") {
-		if len(line) == 0 {
-			// No more text to print, just quit.
-			break
-		}
-
-		for {
-			// Save the attributes of the current cursor helps
-			// us save the text color of the passed error message
-			ansiSaveAttributes()
-			// Print banner with or without the log tag
-			if !tagPrinted {
-				fmt.Print(logBanner)
-				tagPrinted = true
-			} else {
-				fmt.Print(emptyBanner)
-			}
-			// Restore the text color of the error message
-			ansiRestoreAttributes()
-			ansiMoveRight(bannerWidth)
-			// Continue  error message printing
-			fmt.Println(line)
-			break
-		}
-	}
-
-	// Exit because this is a fatal error message
-	os.Exit(1)
-}
-
-type infoMsg struct{}
-
-var info infoMsg
-
-func (i infoMsg) json(msg string, args ...interface{}) {
-	logJSON, err := json.Marshal(&logEntry{
-		Level:   InformationLvl.String(),
-		Message: fmt.Sprintf(msg, args...),
-		Time:    time.Now().UTC().Format(time.RFC3339Nano),
-	})
-	if err != nil {
-		panic(err)
-	}
-	fmt.Println(string(logJSON))
-}
-
-func (i infoMsg) quiet(msg string, args ...interface{}) {
-	i.pretty(msg, args...)
-}
-
-func (i infoMsg) pretty(msg string, args ...interface{}) {
-	c.Printf(msg, args...)
-}
-
-// Info :
-func Info(msg string, data ...interface{}) {
-	consoleLog(info, msg+"\n", data...)
-}
-
-var startupMessage startUpMsg
-
-type startUpMsg struct {
-}
-
-func (s startUpMsg) json(msg string, args ...interface{}) {
-}
-
-func (s startUpMsg) quiet(msg string, args ...interface{}) {
-}
-
-func (s startUpMsg) pretty(msg string, args ...interface{}) {
-	c.Printf(msg, args...)
-}
-
-// StartupMessage :
-func StartupMessage(msg string, data ...interface{}) {
-	consoleLog(startupMessage, msg+"\n", data...)
-}

cmd/logger/message/audit/entry.go

@@ -0,0 +1,92 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package audit
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/handlers"
+)
+
+// Version - represents the current version of audit log structure.
+const Version = "1"
+
+// Entry - audit entry logs.
+type Entry struct {
+	Version      string `json:"version"`
+	DeploymentID string `json:"deploymentid,omitempty"`
+	Time         string `json:"time"`
+	API          struct {
+		Name       string `json:"name,omitempty"`
+		Bucket     string `json:"bucket,omitempty"`
+		Object     string `json:"object,omitempty"`
+		Status     string `json:"status,omitempty"`
+		StatusCode int    `json:"statusCode,omitempty"`
+	} `json:"api"`
+	RemoteHost string                 `json:"remotehost,omitempty"`
+	RequestID  string                 `json:"requestID,omitempty"`
+	UserAgent  string                 `json:"userAgent,omitempty"`
+	ReqClaims  map[string]interface{} `json:"requestClaims,omitempty"`
+	ReqQuery   map[string]string      `json:"requestQuery,omitempty"`
+	ReqHeader  map[string]string      `json:"requestHeader,omitempty"`
+	RespHeader map[string]string      `json:"responseHeader,omitempty"`
+}
+
+// ToEntry - constructs an audit entry object.
+func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int, reqClaims map[string]interface{}) Entry {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	reqQuery := make(map[string]string)
+	for k, v := range r.URL.Query() {
+		reqQuery[k] = strings.Join(v, ",")
+	}
+	reqHeader := make(map[string]string)
+	for k, v := range r.Header {
+		reqHeader[k] = strings.Join(v, ",")
+	}
+	respHeader := make(map[string]string)
+	for k, v := range w.Header() {
+		respHeader[k] = strings.Join(v, ",")
+	}
+	respHeader["Etag"] = strings.Trim(respHeader["Etag"], `"`)
+
+	entry := Entry{
+		Version:      Version,
+		DeploymentID: w.Header().Get("x-minio-deployment-id"),
+		RemoteHost:   handlers.GetSourceIP(r),
+		RequestID:    w.Header().Get("x-amz-request-id"),
+		UserAgent:    r.UserAgent(),
+		Time:         time.Now().UTC().Format(time.RFC3339Nano),
+		ReqQuery:     reqQuery,
+		ReqHeader:    reqHeader,
+		ReqClaims:    reqClaims,
+		RespHeader:   respHeader,
+	}
+
+	entry.API.Name = api
+	entry.API.Bucket = bucket
+	entry.API.Object = object
+	entry.API.Status = http.StatusText(statusCode)
+	entry.API.StatusCode = statusCode
+
+	return entry
+}

cmd/logger/message/log/entry.go

@@ -0,0 +1,50 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package log
+
+// Args - defines the arguments for the API.
+type Args struct {
+	Bucket   string            `json:"bucket,omitempty"`
+	Object   string            `json:"object,omitempty"`
+	Metadata map[string]string `json:"metadata,omitempty"`
+}
+
+// Trace - defines the trace.
+type Trace struct {
+	Message   string            `json:"message,omitempty"`
+	Source    []string          `json:"source,omitempty"`
+	Variables map[string]string `json:"variables,omitempty"`
+}
+
+// API - defines the api type and its args.
+type API struct {
+	Name string `json:"name,omitempty"`
+	Args *Args  `json:"args,omitempty"`
+}
+
+// Entry - defines fields and values of each log entry.
+type Entry struct {
+	DeploymentID string `json:"deploymentid,omitempty"`
+	Level        string `json:"level"`
+	Time         string `json:"time"`
+	API          *API   `json:"api,omitempty"`
+	RemoteHost   string `json:"remotehost,omitempty"`
+	RequestID    string `json:"requestID,omitempty"`
+	UserAgent    string `json:"userAgent,omitempty"`
+	Message      string `json:"message,omitempty"`
+	Trace        *Trace `json:"error,omitempty"`
+}

cmd/logger/reqinfo.go

@@ -35,22 +35,24 @@ type KeyVal struct {
 
 // ReqInfo stores the request info.
 type ReqInfo struct {
-	RemoteHost string   // Client Host/IP
-	UserAgent  string   // User Agent
-	RequestID  string   // x-amz-request-id
-	API        string   // API name - GetObject PutObject NewMultipartUpload etc.
-	BucketName string   // Bucket name
-	ObjectName string   // Object name
-	tags       []KeyVal // Any additional info not accommodated by above fields
+	RemoteHost   string   // Client Host/IP
+	UserAgent    string   // User Agent
+	DeploymentID string   // x-minio-deployment-id
+	RequestID    string   // x-amz-request-id
+	API          string   // API name - GetObject PutObject NewMultipartUpload etc.
+	BucketName   string   // Bucket name
+	ObjectName   string   // Object name
+	tags         []KeyVal // Any additional info not accommodated by above fields
 	sync.RWMutex
 }
 
 // NewReqInfo :
-func NewReqInfo(remoteHost, userAgent, requestID, api, bucket, object string) *ReqInfo {
+func NewReqInfo(remoteHost, userAgent, deploymentID, requestID, api, bucket, object string) *ReqInfo {
 	req := ReqInfo{}
 	req.RemoteHost = remoteHost
 	req.UserAgent = userAgent
 	req.API = api
+	req.DeploymentID = deploymentID
 	req.RequestID = requestID
 	req.BucketName = bucket
 	req.ObjectName = object

cmd/logger/target/console/console.go

@@ -0,0 +1,109 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package console
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/message/log"
+)
+
+// Target implements loggerTarget to send log
+// in plain or json format to the standard output.
+type Target struct{}
+
+// Send log message 'e' to console
+func (c *Target) Send(e interface{}) error {
+	entry, ok := e.(log.Entry)
+	if !ok {
+		return fmt.Errorf("Uexpected log entry structure %#v", e)
+	}
+	if logger.IsJSON() {
+		logJSON, err := json.Marshal(&entry)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(logJSON))
+		return nil
+	}
+
+	trace := make([]string, len(entry.Trace.Source))
+
+	// Add a sequence number and formatting for each stack trace
+	// No formatting is required for the first entry
+	for i, element := range entry.Trace.Source {
+		trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
+	}
+
+	tagString := ""
+	for key, value := range entry.Trace.Variables {
+		if value != "" {
+			if tagString != "" {
+				tagString += ", "
+			}
+			tagString += key + "=" + value
+		}
+	}
+
+	apiString := "API: " + entry.API.Name + "("
+	if entry.API.Args != nil && entry.API.Args.Bucket != "" {
+		apiString = apiString + "bucket=" + entry.API.Args.Bucket
+	}
+	if entry.API.Args != nil && entry.API.Args.Object != "" {
+		apiString = apiString + ", object=" + entry.API.Args.Object
+	}
+	apiString += ")"
+	timeString := "Time: " + time.Now().Format(logger.TimeFormat)
+
+	var requestID string
+	if entry.RequestID != "" {
+		requestID = "\nRequestID: " + entry.RequestID
+	}
+
+	var remoteHost string
+	if entry.RemoteHost != "" {
+		remoteHost = "\nRemoteHost: " + entry.RemoteHost
+	}
+
+	var userAgent string
+	if entry.UserAgent != "" {
+		userAgent = "\nUserAgent: " + entry.UserAgent
+	}
+
+	if len(entry.Trace.Variables) > 0 {
+		tagString = "\n       " + tagString
+	}
+
+	var msg = logger.ColorFgRed(logger.ColorBold(entry.Trace.Message))
+	var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
+		apiString, timeString, requestID, remoteHost, userAgent,
+		msg, tagString, strings.Join(trace, "\n"))
+
+	fmt.Println(output)
+	return nil
+}
+
+// New initializes a new logger target
+// which prints log directly in the standard
+// output.
+func New() *Target {
+	return &Target{}
+}

cmd/logger/target/http/http.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package logger
+package http
 
 import (
 	"bytes"
@@ -22,24 +22,24 @@ import (
 	"errors"
 	"io"
 	"io/ioutil"
-	"net/http"
+	gohttp "net/http"
 )
 
-// HTTPTarget implements loggerTarget and sends the json
+// Target implements logger.Target and sends the json
 // format of a log entry to the configured http endpoint.
 // An internal buffer of logs is maintained but when the
 // buffer is full, new logs are just ignored and an error
 // is returned to the caller.
-type HTTPTarget struct {
+type Target struct {
 	// Channel of log entries
 	logCh chan interface{}
 
 	// HTTP(s) endpoint
 	endpoint string
-	client   http.Client
+	client   gohttp.Client
 }
 
-func (h *HTTPTarget) startHTTPLogger() {
+func (h *Target) startHTTPLogger() {
 	// Create a routine which sends json logs received
 	// from an internal channel.
 	go func() {
@@ -49,7 +49,7 @@ func (h *HTTPTarget) startHTTPLogger() {
 				continue
 			}
 
-			req, err := http.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
+			req, err := gohttp.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
 			req.Header.Set("Content-Type", "application/json")
 
 			resp, err := h.client.Do(req)
@@ -65,12 +65,12 @@ func (h *HTTPTarget) startHTTPLogger() {
 	}()
 }
 
-// NewHTTP initializes a new logger target which
+// New initializes a new logger target which
 // sends log over http to the specified endpoint
-func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
-	h := HTTPTarget{
+func New(endpoint string, transport *gohttp.Transport) *Target {
+	h := Target{
 		endpoint: endpoint,
-		client: http.Client{
+		client: gohttp.Client{
 			Transport: transport,
 		},
 		logCh: make(chan interface{}, 10000),
@@ -80,7 +80,8 @@ func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
 	return &h
 }
 
-func (h *HTTPTarget) send(entry interface{}) error {
+// Send log message 'e' to http target.
+func (h *Target) Send(entry interface{}) error {
 	select {
 	case h.logCh <- entry:
 	default:

cmd/logger/targets.go

@@ -16,18 +16,18 @@
 
 package logger
 
-// LoggingTarget is the entity that we will receive
-// a single log entry and send it to the log target
-//   e.g. send the log to a http server
-type LoggingTarget interface {
-	send(entry interface{}) error
+// Target is the entity that we will receive
+// a single log entry and Send it to the log target
+//   e.g. Send the log to a http server
+type Target interface {
+	Send(entry interface{}) error
 }
 
 // Targets is the set of enabled loggers
-var Targets = []LoggingTarget{}
+var Targets = []Target{}
 
 // AddTarget adds a new logger target to the
 // list of enabled loggers
-func AddTarget(t LoggingTarget) {
+func AddTarget(t Target) {
 	Targets = append(Targets, t)
 }

cmd/logger/utils.go

@@ -33,25 +33,25 @@ var (
 		return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
 	}
 
-	colorBold = func() func(a ...interface{}) string {
+	ColorBold = func() func(a ...interface{}) string {
 		if isTerminal() {
 			return color.New(color.Bold).SprintFunc()
 		}
 		return fmt.Sprint
 	}()
-	colorFgRed = func() func(format string, a ...interface{}) string {
+	ColorFgRed = func() func(format string, a ...interface{}) string {
 		if isTerminal() {
 			return color.New(color.FgRed).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
-	colorBgRed = func() func(format string, a ...interface{}) string {
+	ColorBgRed = func() func(format string, a ...interface{}) string {
 		if isTerminal() {
 			return color.New(color.BgRed).SprintfFunc()
 		}
 		return fmt.Sprintf
 	}()
-	colorFgWhite = func() func(format string, a ...interface{}) string {
+	ColorFgWhite = func() func(format string, a ...interface{}) string {
 		if isTerminal() {
 			return color.New(color.FgWhite).SprintfFunc()
 		}
@@ -83,17 +83,5 @@ func ansiRestoreAttributes() {
 	if isTerminal() {
 		ansiEscape("8")
 	}
-}
 
-func uniqueEntries(paths []string) []string {
-	found := map[string]bool{}
-	unqiue := []string{}
-
-	for v := range paths {
-		if _, ok := found[paths[v]]; !ok {
-			found[paths[v]] = true
-			unqiue = append(unqiue, paths[v])
-		}
-	}
-	return unqiue
 }

cmd/object-api-datatypes.go

@@ -108,6 +108,8 @@ type ObjectInfo struct {
 	// Implements writer and reader used by CopyObject API
 	Writer       io.WriteCloser `json:"-"`
 	Reader       *hash.Reader   `json:"-"`
+	PutObjReader *PutObjReader  `json:"-"`
+
 	metadataOnly bool
 
 	// Date and time when the object was last accessed.

cmd/object-api-deleteobject_test.go

@@ -92,7 +92,7 @@ func testDeleteObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 		for _, object := range testCase.objectToUploads {
 			md5Bytes := md5.Sum([]byte(object.content))
-			_, err = obj.PutObject(context.Background(), testCase.bucketName, object.name, mustGetHashReader(t, bytes.NewBufferString(object.content),
+			_, err = obj.PutObject(context.Background(), testCase.bucketName, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
 				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), nil, ObjectOptions{})
 			if err != nil {
 				t.Fatalf("%s : %s", instanceType, err.Error())

cmd/object-api-getobject_test.go

@@ -75,7 +75,7 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// iterate through the above set of inputs and upkoad the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -220,7 +220,7 @@ func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks [
 	// iterate through the above set of inputs and upkoad the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -333,7 +333,7 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str
 	// iterate through the above set of inputs and upkoad the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)

cmd/object-api-getobjectinfo_test.go

@@ -35,13 +35,13 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 	opts := ObjectOptions{}
-	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/asiapics.jpg", mustGetHashReader(t, bytes.NewBufferString("asiapics"), int64(len("asiapics")), "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/asiapics.jpg", mustGetPutObjReader(t, bytes.NewBufferString("asiapics"), int64(len("asiapics")), "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
 	// Put an empty directory
-	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/empty-dir/", mustGetHashReader(t, bytes.NewBufferString(""), int64(len("")), "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/empty-dir/", mustGetPutObjReader(t, bytes.NewBufferString(""), int64(len("")), "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}

cmd/object-api-interface.go

@@ -22,7 +22,6 @@ import (
 	"net/http"
 
 	"github.com/minio/minio-go/pkg/encrypt"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -66,7 +65,7 @@ type ObjectLayer interface {
 	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (reader *GetObjectReader, err error)
 	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 
@@ -75,10 +74,10 @@ type ObjectLayer interface {
 	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
 	CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int,
 		startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error)
-	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
+	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error)
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
-	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
+	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
 
 	// Healing operations.
 	ReloadFormat(ctx context.Context, dryRun bool) error

cmd/object-api-listobjects_test.go

@@ -70,7 +70,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 	for _, object := range testObjects {
 		md5Bytes := md5.Sum([]byte(object.content))
-		_, err = obj.PutObject(context.Background(), testBuckets[0], object.name, mustGetHashReader(t, bytes.NewBufferString(object.content),
+		_, err = obj.PutObject(context.Background(), testBuckets[0], object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
 			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), object.meta, ObjectOptions{})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
@@ -617,7 +617,7 @@ func BenchmarkListObjects(b *testing.B) {
 	// Insert objects to be listed and benchmarked later.
 	for i := 0; i < 20000; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject(context.Background(), bucket, key, mustGetHashReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), nil, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), bucket, key, mustGetPutObjReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), nil, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}

cmd/object-api-multipart_test.go

@@ -221,7 +221,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
+		_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -235,7 +235,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 
 	// Object part upload should fail with quorum not available.
 	testCase := createPartCases[len(createPartCases)-1]
-	_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
+	_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
 	if err == nil {
 		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
 	}
@@ -354,7 +354,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 
 	// Validate all the test cases.
 	for i, testCase := range testCases {
-		actualInfo, actualErr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, testCase.inputSHA256), opts)
+		actualInfo, actualErr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, testCase.inputSHA256), opts)
 		// All are test cases above are expected to fail.
 		if actualErr != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
@@ -488,7 +488,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
+		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -1309,7 +1309,7 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
+		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -1550,7 +1550,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
+		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -1801,7 +1801,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, part := range parts {
-		_, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, sha256sum), opts)
+		_, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, mustGetPutObjReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, sha256sum), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err)
 		}
@@ -1904,7 +1904,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 	}
 
 	for i, testCase := range testCases {
-		actualResult, actualErr := obj.CompleteMultipartUpload(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.parts)
+		actualResult, actualErr := obj.CompleteMultipartUpload(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.parts, ObjectOptions{})
 		if actualErr != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, actualErr)
 		}

cmd/object-api-putobject_test.go

@@ -172,7 +172,7 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 	}
 
 	for i, testCase := range testCases {
-		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), testCase.inputMeta, ObjectOptions{})
+		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), testCase.inputMeta, ObjectOptions{})
 		if actualErr != nil && testCase.expectedError == nil {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error())
 		}
@@ -245,7 +245,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di
 
 	sha256sum := ""
 	for i, testCase := range testCases {
-		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{})
+		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{})
 		if actualErr != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
 		}
@@ -294,7 +294,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di
 		InsufficientWriteQuorum{},
 	}
 
-	_, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{})
+	_, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{})
 	if actualErr != nil && testCase.shouldPass {
 		t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", len(testCases)+1, instanceType, actualErr.Error())
 	}
@@ -326,7 +326,7 @@ func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disk
 
 	data := []byte("hello, world")
 	// Create object.
-	_, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		// Failed to create object, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -371,7 +371,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
 	md5Writer.Write(fiveMBBytes)
 	etag1 := hex.EncodeToString(md5Writer.Sum(nil))
 	sha256sum := ""
-	_, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), int64(len(fiveMBBytes)), etag1, sha256sum), opts)
+	_, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), int64(len(fiveMBBytes)), etag1, sha256sum), opts)
 	if err != nil {
 		// Failed to upload object part, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -382,7 +382,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
 	md5Writer = md5.New()
 	md5Writer.Write(data)
 	etag2 := hex.EncodeToString(md5Writer.Sum(nil))
-	_, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), etag2, sha256sum), opts)
+	_, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 2, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), etag2, sha256sum), opts)
 	if err != nil {
 		// Failed to upload object part, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -393,7 +393,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
 		{ETag: etag1, PartNumber: 1},
 		{ETag: etag2, PartNumber: 2},
 	}
-	_, err = obj.CompleteMultipartUpload(context.Background(), bucket, object, uploadID, parts)
+	_, err = obj.CompleteMultipartUpload(context.Background(), bucket, object, uploadID, parts, ObjectOptions{})
 	if err != nil {
 		// Failed to complete multipart upload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())

cmd/object-api-utils.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"fmt"
@@ -35,6 +36,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
+	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/wildcard"
 	"github.com/skyrings/skyring-common/tools/uuid"
@@ -597,3 +599,63 @@ func (g *GetObjectReader) Read(p []byte) (n int, err error) {
 	}
 	return
 }
+
+//SealMD5CurrFn seals md5sum with object encryption key and returns sealed
+// md5sum
+type SealMD5CurrFn func([]byte) []byte
+
+// PutObjReader is a type that wraps sio.EncryptReader and
+// underlying hash.Reader in a struct
+type PutObjReader struct {
+	*hash.Reader              // actual data stream
+	rawReader    *hash.Reader // original data stream
+	sealMD5Fn    SealMD5CurrFn
+}
+
+// Size returns the absolute number of bytes the Reader
+// will return during reading. It returns -1 for unlimited
+// data.
+func (p *PutObjReader) Size() int64 {
+	return p.Reader.Size()
+}
+
+// MD5CurrentHexString returns the current MD5Sum or encrypted MD5Sum
+// as a hex encoded string
+func (p *PutObjReader) MD5CurrentHexString() string {
+	md5sumCurr := p.rawReader.MD5Current()
+	if p.sealMD5Fn != nil {
+		md5sumCurr = p.sealMD5Fn(md5sumCurr)
+	}
+	return hex.EncodeToString(md5sumCurr)
+}
+
+// NewPutObjReader returns a new PutObjReader and holds
+// reference to underlying data stream from client and the encrypted
+// data reader
+func NewPutObjReader(rawReader *hash.Reader, encReader *hash.Reader, encKey []byte) *PutObjReader {
+	p := PutObjReader{Reader: rawReader, rawReader: rawReader}
+
+	if len(encKey) != 0 && encReader != nil {
+		var objKey crypto.ObjectKey
+		copy(objKey[:], encKey)
+		p.sealMD5Fn = sealETagFn(objKey)
+		p.Reader = encReader
+	}
+
+	return &p
+}
+
+func sealETag(encKey crypto.ObjectKey, md5CurrSum []byte) []byte {
+	var emptyKey [32]byte
+	if bytes.Equal(encKey[:], emptyKey[:]) {
+		return md5CurrSum
+	}
+	return encKey.SealETag(md5CurrSum)
+}
+
+func sealETagFn(key crypto.ObjectKey) SealMD5CurrFn {
+	fn := func(md5sumcurr []byte) []byte {
+		return sealETag(key, md5sumcurr)
+	}
+	return fn
+}

cmd/object-handlers-common.go

@@ -75,7 +75,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
 			if !ifModifiedSince(objInfo.ModTime, givenTime) {
 				// If the object is not modified since the specified time.
 				writeHeaders()
-				writeErrorResponse(w, ErrPreconditionFailed, r.URL)
+				writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
 				return true
 			}
 		}
@@ -89,7 +89,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
 			if ifModifiedSince(objInfo.ModTime, givenTime) {
 				// If the object is modified since the specified time.
 				writeHeaders()
-				writeErrorResponse(w, ErrPreconditionFailed, r.URL)
+				writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
 				return true
 			}
 		}
@@ -102,7 +102,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
 		if objInfo.ETag != "" && !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
 			// If the object ETag does not match with the specified ETag.
 			writeHeaders()
-			writeErrorResponse(w, ErrPreconditionFailed, r.URL)
+			writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
 			return true
 		}
 	}
@@ -114,7 +114,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
 		if objInfo.ETag != "" && isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) {
 			// If the object ETag matches with the specified ETag.
 			writeHeaders()
-			writeErrorResponse(w, ErrPreconditionFailed, r.URL)
+			writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
 			return true
 		}
 	}
@@ -174,7 +174,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectIn
 			if ifModifiedSince(objInfo.ModTime, givenTime) {
 				// If the object is modified since the specified time.
 				writeHeaders()
-				writeErrorResponse(w, ErrPreconditionFailed, r.URL)
+				writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
 				return true
 			}
 		}
@@ -187,7 +187,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectIn
 		if !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
 			// If the object ETag does not match with the specified ETag.
 			writeHeaders()
-			writeErrorResponse(w, ErrPreconditionFailed, r.URL)
+			writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
 			return true
 		}
 	}

cmd/object-handlers_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/xml"
 	"fmt"
 	"io"
@@ -83,7 +84,7 @@ func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{})
+		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -355,7 +356,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{})
+		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -1391,7 +1392,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 	for i, input := range putObjectInputs {
 		// uploading the object.
 		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName,
-			mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
+			mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -1449,7 +1450,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 		})
 	}
 
-	result, err := obj.CompleteMultipartUpload(context.Background(), bucketName, testObject, uploadID, parts)
+	result, err := obj.CompleteMultipartUpload(context.Background(), bucketName, testObject, uploadID, parts, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Test: %s complete multipart upload failed: <ERROR> %v", instanceType, err)
 	}
@@ -1501,7 +1502,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -1811,9 +1812,13 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	// this is required even to assert the copied object,
 	bytesData := []struct {
 		byteData []byte
+		md5sum   string
 	}{
-		{generateBytesData(6 * humanize.KiByte)},
+		{byteData: generateBytesData(6 * humanize.KiByte)},
 	}
+	h := md5.New()
+	h.Write(bytesData[0].byteData)
+	bytesData[0].md5sum = hex.EncodeToString(h.Sum(nil))
 
 	buffers := []*bytes.Buffer{
 		new(bytes.Buffer),
@@ -1826,23 +1831,29 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		objectName    string
 		contentLength int64
 		textData      []byte
+		md5sum        string
 		metaData      map[string]string
 	}{
 		// case - 1.
-		{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
+		{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, bytesData[0].md5sum, make(map[string]string)},
 
 		// case - 2.
 		// used for anonymous HTTP request test.
-		{bucketName, anonObject, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
+		{bucketName, anonObject, int64(len(bytesData[0].byteData)), bytesData[0].byteData, bytesData[0].md5sum, make(map[string]string)},
 	}
+
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
+		var objInfo ObjectInfo
+		objInfo, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
 		}
+		if objInfo.ETag != input.md5sum {
+			t.Fatalf("Put Object case %d:  Checksum mismatched: <ERROR> got %s, expected %s", i+1, input.md5sum, objInfo.ETag)
+		}
 	}
 
 	// test cases with inputs and expected result for Copy Object.
@@ -2105,6 +2116,16 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			t.Fatalf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
 		}
 		if rec.Code == http.StatusOK {
+			var cpObjResp CopyObjectResponse
+			if err = xml.Unmarshal(rec.Body.Bytes(), &cpObjResp); err != nil {
+				t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i+1, instanceType, err)
+			}
+
+			retag := canonicalizeETag(cpObjResp.ETag)
+			if retag != bytesData[0].md5sum {
+				t.Fatalf("Test %d: %s: Failed to CopyObject: <ERROR> got %s, expected %s", i+1, instanceType, retag, bytesData[0].md5sum)
+			}
+
 			// See if the new object is formed.
 			// testing whether the copy was successful.
 			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
@@ -2462,7 +2483,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, part := range parts {
 		_, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID,
-			mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""), opts)
+			mustGetPutObjReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err)
 		}
@@ -2813,7 +2834,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri
 	// Iterating over createPartCases to generate multipart chunks.
 	for _, part := range parts {
 		_, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID,
-			mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""), opts)
+			mustGetPutObjReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err)
 		}
@@ -2951,7 +2972,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -3657,7 +3678,7 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
 	uploadIDCopy := uploadID
 
 	// create an object Part, will be used to test list object parts.
-	_, err = obj.PutObjectPart(context.Background(), bucketName, testObject, uploadID, 1, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "5d41402abc4b2a76b9719d911017c592", ""), opts)
+	_, err = obj.PutObjectPart(context.Background(), bucketName, testObject, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "5d41402abc4b2a76b9719d911017c592", ""), opts)
 	if err != nil {
 		t.Fatalf("Minio %s : %s.", instanceType, err)
 	}

cmd/object_api_suite_test.go

@@ -106,7 +106,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr
 		expectedETaghex := getMD5Hash(data)
 
 		var calcPartInfo PartInfo
-		calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBuffer(data), int64(len(data)), expectedETaghex, ""), opts)
+		calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetPutObjReader(t, bytes.NewBuffer(data), int64(len(data)), expectedETaghex, ""), opts)
 		if err != nil {
 			t.Errorf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -118,7 +118,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr
 			ETag:       calcPartInfo.ETag,
 		})
 	}
-	objInfo, err := obj.CompleteMultipartUpload(context.Background(), "bucket", "key", uploadID, completedParts.Parts)
+	objInfo, err := obj.CompleteMultipartUpload(context.Background(), "bucket", "key", uploadID, completedParts.Parts, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -157,7 +157,7 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan
 
 		metadata["md5"] = expectedETaghex
 		var calcPartInfo PartInfo
-		calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), expectedETaghex, ""), opts)
+		calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetPutObjReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), expectedETaghex, ""), opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -199,7 +199,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
 		metadata := make(map[string]string)
 		metadata["etag"] = expectedETaghex
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), metadata, opts)
+		objInfo, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), metadata, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -253,7 +253,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// check before paging occurs.
 	for i := 0; i < 5; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -273,7 +273,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// check after paging occurs pages work.
 	for i := 6; i <= 10; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -290,11 +290,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 	// check paging with prefix at end returns less objects.
 	{
-		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
-		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix2", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix2", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -332,11 +332,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 	// check delimited results with delimiter and prefix.
 	{
-		_, err = obj.PutObject(context.Background(), "bucket", "this/is/delimited", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", "this/is/delimited", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
-		_, err = obj.PutObject(context.Background(), "bucket", "this/is/also/a/delimited/file", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", "this/is/also/a/delimited/file", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -448,14 +448,14 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan
 	var opts ObjectOptions
 	uploadContent := "The list of parts was not in ascending order. The parts list must be specified in order by part number."
 	length := int64(len(uploadContent))
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
 	uploadContent = "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
 	length = int64(len(uploadContent))
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -478,7 +478,7 @@ func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(t *testing.T) {
 // Tests validate that bucket operation on non-existent bucket fails.
 func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	var opts ObjectOptions
-	_, err := obj.PutObject(context.Background(), "bucket1", "object", mustGetHashReader(t, bytes.NewBufferString("one"), int64(len("one")), "", ""), nil, opts)
+	_, err := obj.PutObject(context.Background(), "bucket1", "object", mustGetPutObjReader(t, bytes.NewBufferString("one"), int64(len("one")), "", ""), nil, opts)
 	if err == nil {
 		t.Fatal("Expected error but found nil")
 	}
@@ -526,7 +526,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 	var bytesBuffer1 bytes.Buffer
 	var opts ObjectOptions
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerEOF, length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, readerEOF, length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -539,7 +539,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	var bytesBuffer2 bytes.Buffer
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerNoEOF, length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, readerNoEOF, length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -568,7 +568,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle
 	uploadContent := `The specified multipart upload does not exist. The upload ID might be invalid, or the multipart
  upload might have been aborted or completed.`
 	length := int64(len(uploadContent))
-	_, err = obj.PutObject(context.Background(), "bucket", "dir1/dir2/object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "dir1/dir2/object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -741,7 +741,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
 	content := "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."
 	length := int64(len(content))
 	var opts ObjectOptions
-	_, err = obj.PutObject(context.Background(), bucketName, "dir1/dir3/object", mustGetHashReader(t, bytes.NewBufferString(content), length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), bucketName, "dir1/dir3/object", mustGetPutObjReader(t, bytes.NewBufferString(content), length, "", ""), nil, opts)
 
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
@@ -783,7 +783,7 @@ func testContentType(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	var opts ObjectOptions
 	uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
 	// Test empty.
-	_, err = obj.PutObject(context.Background(), "bucket", "minio.png", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "minio.png", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}

cmd/posix-list-dir_other.go

@@ -1,4 +1,4 @@
-// +build plan9
+// +build plan9 solaris
 
 /*
  * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.

cmd/posix-list-dir_unix.go

@@ -1,4 +1,4 @@
-// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris
+// +build darwin dragonfly freebsd linux nacl netbsd openbsd
 
 /*
  * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.

cmd/posix.go

@@ -83,8 +83,10 @@ func checkPathLength(pathName string) error {
 		return errFileNameTooLong
 	}
 
-	// Convert any '\' to '/'.
-	pathName = filepath.ToSlash(pathName)
+	if runtime.GOOS == "windows" {
+		// Convert any '\' to '/'.
+		pathName = filepath.ToSlash(pathName)
+	}
 
 	// Check each path segment length is > 255
 	for len(pathName) > 0 && pathName != "." && pathName != "/" {
@@ -374,6 +376,7 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 		default:
 			fi, err := os.Stat(entry)
 			if err != nil {
+				err = osErrToFSFileErr(err)
 				return err
 			}
 			atomic.AddUint64(&s.totalUsed, uint64(fi.Size()))
@@ -413,6 +416,7 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 				default:
 					fi, err := os.Stat(entry)
 					if err != nil {
+						err = osErrToFSFileErr(err)
 						return err
 					}
 					usage = usage + uint64(fi.Size())

cmd/prepare-storage.go

@@ -210,7 +210,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointLi
 		}
 	}
 
-	logger.SetDeploymentID(format.ID)
+	globalDeploymentID = format.ID
 
 	if err = formatXLFixLocalDeploymentID(context.Background(), storageDisks, format); err != nil {
 		return nil, err

cmd/rest/client.go

@@ -35,9 +35,10 @@ const DefaultRESTTimeout = 1 * time.Minute
 
 // Client - http based RPC client.
 type Client struct {
-	httpClient   *http.Client
-	url          *url.URL
-	newAuthToken func() string
+	httpClient          *http.Client
+	httpIdleConnsCloser func()
+	url                 *url.URL
+	newAuthToken        func() string
 }
 
 // Call - make a REST call.
@@ -67,6 +68,13 @@ func (c *Client) Call(method string, values url.Values, body io.Reader) (reply i
 	return resp.Body, nil
 }
 
+// Close closes all idle connections of the underlying http client
+func (c *Client) Close() {
+	if c.httpIdleConnsCloser != nil {
+		c.httpIdleConnsCloser()
+	}
+}
+
 func newCustomDialContext(timeout time.Duration) func(ctx context.Context, network, addr string) (net.Conn, error) {
 	return func(ctx context.Context, network, addr string) (net.Conn, error) {
 		dialer := &net.Dialer{
@@ -84,25 +92,26 @@ func newCustomDialContext(timeout time.Duration) func(ctx context.Context, netwo
 	}
 }
 
-// NewClient - returns new RPC client.
+// NewClient - returns new REST client.
 func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAuthToken func() string) *Client {
+	// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
+	// except custom DialContext and TLSClientConfig.
+	tr := &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           newCustomDialContext(timeout),
+		MaxIdleConnsPerHost:   4096,
+		MaxIdleConns:          4096,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       tlsConfig,
+		DisableCompression:    true,
+	}
+
 	return &Client{
-		httpClient: &http.Client{
-			// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
-			// except custom DialContext and TLSClientConfig.
-			Transport: &http.Transport{
-				Proxy:                 http.ProxyFromEnvironment,
-				DialContext:           newCustomDialContext(timeout),
-				MaxIdleConnsPerHost:   4096,
-				MaxIdleConns:          4096,
-				IdleConnTimeout:       90 * time.Second,
-				TLSHandshakeTimeout:   10 * time.Second,
-				ExpectContinueTimeout: 1 * time.Second,
-				TLSClientConfig:       tlsConfig,
-				DisableCompression:    true,
-			},
-		},
-		url:          url,
-		newAuthToken: newAuthToken,
+		httpClient:          &http.Client{Transport: tr},
+		httpIdleConnsCloser: tr.CloseIdleConnections,
+		url:                 url,
+		newAuthToken:        newAuthToken,
 	}
 }

cmd/routers.go

@@ -47,8 +47,8 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 
 // List of some generic handlers which are applied for all incoming requests.
 var globalHandlers = []HandlerFunc{
-	// set x-amz-request-id header.
-	addrequestIDHeader,
+	// set x-amz-request-id, x-minio-deployment-id header.
+	addCustomHeaders,
 	// set HTTP security headers such as Content-Security-Policy.
 	addSecurityHeaders,
 	// Forward path style requests to actual host in a bucket federated setup.

cmd/server-main.go

@@ -41,13 +41,13 @@ var serverFlags = []cli.Flag{
 	cli.StringFlag{
 		Name:  "address",
 		Value: ":" + globalMinioPort,
-		Usage: "Bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname.",
+		Usage: "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
 	},
 }
 
 var serverCmd = cli.Command{
 	Name:   "server",
-	Usage:  "Start object storage server.",
+	Usage:  "start object storage server",
 	Flags:  append(serverFlags, globalFlags...),
 	Action: serverMain,
 	CustomHelpTemplate: `NAME:
@@ -112,10 +112,10 @@ EXAMPLES:
   4. Start erasure coded minio server on a node with 64 drives.
      $ {{.HelpName}} /mnt/export{1...64}
 
-  5. Start distributed minio server on an 8 node setup with 8 drives each. Run following command on all the 8 nodes.
+  5. Start distributed minio server on an 32 node setup with 32 drives each. Run following command on all the 32 nodes.
      $ export MINIO_ACCESS_KEY=minio
      $ export MINIO_SECRET_KEY=miniostorage
-     $ {{.HelpName}} http://node{1...8}.example.com/mnt/export/{1...8}
+     $ {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32}
 
   6. Start minio server with edge caching enabled.
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"

cmd/storage-class_test.go

@@ -189,7 +189,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 
 	// Object for test case 1 - No StorageClass defined, no MetaData in PutObject
 	object1 := "object1"
-	_, err = obj.PutObject(context.Background(), bucket, object1, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), bucket, object1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -200,7 +200,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object2 := "object2"
 	metadata2 := make(map[string]string)
 	metadata2["x-amz-storage-class"] = reducedRedundancyStorageClass
-	_, err = obj.PutObject(context.Background(), bucket, object2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata2, opts)
+	_, err = obj.PutObject(context.Background(), bucket, object2, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata2, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -211,7 +211,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object3 := "object3"
 	metadata3 := make(map[string]string)
 	metadata3["x-amz-storage-class"] = standardStorageClass
-	_, err = obj.PutObject(context.Background(), bucket, object3, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata3, opts)
+	_, err = obj.PutObject(context.Background(), bucket, object3, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata3, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -227,7 +227,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		Scheme: "EC",
 	}
 
-	_, err = obj.PutObject(context.Background(), bucket, object4, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata4, opts)
+	_, err = obj.PutObject(context.Background(), bucket, object4, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata4, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -245,7 +245,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		Scheme: "EC",
 	}
 
-	_, err = obj.PutObject(context.Background(), bucket, object5, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata5, opts)
+	_, err = obj.PutObject(context.Background(), bucket, object5, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata5, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -263,7 +263,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		Scheme: "EC",
 	}
 
-	_, err = obj.PutObject(context.Background(), bucket, object6, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata6, opts)
+	_, err = obj.PutObject(context.Background(), bucket, object6, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata6, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -281,7 +281,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		Scheme: "EC",
 	}
 
-	_, err = obj.PutObject(context.Background(), bucket, object7, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata7, opts)
+	_, err = obj.PutObject(context.Background(), bucket, object7, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata7, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}

cmd/storage-rest-client.go

@@ -322,6 +322,7 @@ func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPa
 // Close - marks the client as closed.
 func (client *storageRESTClient) Close() error {
 	client.connected = false
+	client.restClient.Close()
 	return nil
 }
 

cmd/storage-rest-common.go

@@ -16,7 +16,7 @@
 
 package cmd
 
-const storageRESTVersion = "v1"
+const storageRESTVersion = "v2"
 const storageRESTPath = minioReservedBucketPath + "/storage/" + storageRESTVersion + "/"
 
 const (

cmd/storage-rest-server.go

@@ -378,5 +378,6 @@ func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
 			Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)
 
+		subrouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 	}
 }

cmd/sts-handlers.go

@@ -127,6 +127,8 @@ type ClientGrantsResult struct {
 func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AssumeRoleWithClientGrants")
 
+	defer logger.AuditLog(w, r, "AssumeRoleWithClientGrants", nil)
+
 	if globalIAMValidators == nil {
 		writeSTSErrorResponse(w, ErrSTSNotInitialized)
 		return

cmd/test-utils_test.go

@@ -136,12 +136,12 @@ func calculateSignedChunkLength(chunkDataSize int64) int64 {
 		2 // CRLF
 }
 
-func mustGetHashReader(t TestErrHandler, data io.Reader, size int64, md5hex, sha256hex string) *hash.Reader {
+func mustGetPutObjReader(t TestErrHandler, data io.Reader, size int64, md5hex, sha256hex string) *PutObjReader {
 	hr, err := hash.NewReader(data, size, md5hex, sha256hex, size)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return hr
+	return NewPutObjReader(hr, nil, nil)
 }
 
 // calculateSignedChunkLength - calculates the length of the overall stream (data + metadata)

cmd/update-main.go

@@ -40,12 +40,12 @@ import (
 // Check for new software updates.
 var updateCmd = cli.Command{
 	Name:   "update",
-	Usage:  "Check for a new software update.",
+	Usage:  "update minio to latest release",
 	Action: mainUpdate,
 	Flags: []cli.Flag{
 		cli.BoolFlag{
 			Name:  "quiet",
-			Usage: "Disable any update prompt message.",
+			Usage: "disable any update prompt message",
 		},
 	},
 	CustomHelpTemplate: `Name:

cmd/utils.go

@@ -397,12 +397,13 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
 		object = prefix
 	}
 	reqInfo := &logger.ReqInfo{
-		RequestID:  w.Header().Get(responseRequestIDKey),
-		RemoteHost: handlers.GetSourceIP(r),
-		UserAgent:  r.UserAgent(),
-		API:        api,
-		BucketName: bucket,
-		ObjectName: object,
+		DeploymentID: w.Header().Get(responseDeploymentIDKey),
+		RequestID:    w.Header().Get(responseRequestIDKey),
+		RemoteHost:   handlers.GetSourceIP(r),
+		UserAgent:    r.UserAgent(),
+		API:          api,
+		BucketName:   bucket,
+		ObjectName:   object,
 	}
 	return logger.SetReqInfo(context.Background(), reqInfo)
 }

cmd/version-main.go

@@ -23,7 +23,7 @@ import (
 
 var versionCmd = cli.Command{
 	Name:   "version",
-	Usage:  "Print version.",
+	Usage:  "print version",
 	Action: mainVersion,
 	CustomHelpTemplate: `NAME:
    {{.HelpName}} - {{.Usage}}

cmd/web-handlers.go

@@ -707,7 +707,7 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs,
 func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "WebUpload")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "WebUpload", mustGetClaimsFromToken(r))
 
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
@@ -768,7 +768,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	// Extract incoming metadata if any.
 	metadata, err := extractMetadata(context.Background(), r)
 	if err != nil {
-		writeErrorResponse(w, ErrInternalError, r.URL)
+		writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -807,7 +807,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		writeWebErrorResponse(w, err)
 		return
 	}
-	opts := ObjectOptions{}
+	var opts ObjectOptions
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
 		if _, err = objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil {
@@ -816,7 +816,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	objInfo, err := putObject(ctx, bucket, object, hashReader, metadata, opts)
+	objInfo, err := putObject(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), metadata, opts)
 	if err != nil {
 		writeWebErrorResponse(w, err)
 		return
@@ -845,7 +845,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "WebDownload")
 
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "WebDownload", mustGetClaimsFromToken(r))
 
 	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()
@@ -916,7 +916,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if _, err = DecryptObjectInfo(objInfo, r.Header); err != nil {
+		if _, err = DecryptObjectInfo(&objInfo, r.Header); err != nil {
 			writeWebErrorResponse(w, err)
 			return
 		}
@@ -1027,7 +1027,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 	}
 
 	ctx := newContext(r, w, "WebDownloadZip")
-	defer logger.AuditLog(ctx, w, r)
+	defer logger.AuditLog(w, r, "WebDownloadZip", mustGetClaimsFromToken(r))
 
 	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()
@@ -1113,7 +1113,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 			}
 			length = info.Size
 			if objectAPI.IsEncryptionSupported() {
-				if _, err = DecryptObjectInfo(info, r.Header); err != nil {
+				if _, err = DecryptObjectInfo(&info, r.Header); err != nil {
 					writeWebErrorResponse(w, err)
 					return err
 				}

cmd/web-handlers_test.go

@@ -349,7 +349,7 @@ func testDeleteBucketWebHandler(obj ObjectLayer, instanceType string, t TestErrH
 	for _, test := range testCases {
 		if test.initWithObject {
 			data := bytes.NewBufferString("hello")
-			_, err = obj.PutObject(context.Background(), test.bucketName, "object", mustGetHashReader(t, data, int64(data.Len()), "", ""), nil, opts)
+			_, err = obj.PutObject(context.Background(), test.bucketName, "object", mustGetPutObjReader(t, data, int64(data.Len()), "", ""), nil, opts)
 			// _, err = obj.PutObject(test.bucketName, "object", int64(data.Len()), data, nil, "")
 			if err != nil {
 				t.Fatalf("could not put object to %s, %s", test.bucketName, err.Error())
@@ -485,7 +485,7 @@ func testListObjectsWebHandler(obj ObjectLayer, instanceType string, t TestErrHa
 
 	data := bytes.Repeat([]byte("a"), objectSize)
 	metadata := map[string]string{"etag": "c9a34cfc85d982698c6ac89f76071abd"}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{})
 
 	if err != nil {
 		t.Fatalf("Was not able to upload an object, %v", err)
@@ -589,14 +589,14 @@ func testRemoveObjectWebHandler(obj ObjectLayer, instanceType string, t TestErrH
 
 	data := bytes.Repeat([]byte("a"), objectSize)
 	metadata := map[string]string{"etag": "c9a34cfc85d982698c6ac89f76071abd"}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Was not able to upload an object, %v", err)
 	}
 
 	objectName = "a/object"
 	metadata = map[string]string{"etag": "c9a34cfc85d982698c6ac89f76071abd"}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Was not able to upload an object, %v", err)
 	}
@@ -987,7 +987,7 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl
 
 	content := []byte("temporary file's content")
 	metadata := map[string]string{"etag": "01ce59706106fe5e02e7f55fffda7f34"}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(content), int64(len(content)), metadata["etag"], ""), metadata, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(content), int64(len(content)), metadata["etag"], ""), metadata, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Was not able to upload an object, %v", err)
 	}
@@ -1090,9 +1090,9 @@ func testWebHandlerDownloadZip(obj ObjectLayer, instanceType string, t TestErrHa
 		t.Fatalf("%s : %s", instanceType, err)
 	}
 
-	obj.PutObject(context.Background(), bucket, "a/one", mustGetHashReader(t, strings.NewReader(fileOne), int64(len(fileOne)), "", ""), nil, opts)
-	obj.PutObject(context.Background(), bucket, "a/b/two", mustGetHashReader(t, strings.NewReader(fileTwo), int64(len(fileTwo)), "", ""), nil, opts)
-	obj.PutObject(context.Background(), bucket, "a/c/three", mustGetHashReader(t, strings.NewReader(fileThree), int64(len(fileThree)), "", ""), nil, opts)
+	obj.PutObject(context.Background(), bucket, "a/one", mustGetPutObjReader(t, strings.NewReader(fileOne), int64(len(fileOne)), "", ""), nil, opts)
+	obj.PutObject(context.Background(), bucket, "a/b/two", mustGetPutObjReader(t, strings.NewReader(fileTwo), int64(len(fileTwo)), "", ""), nil, opts)
+	obj.PutObject(context.Background(), bucket, "a/c/three", mustGetPutObjReader(t, strings.NewReader(fileThree), int64(len(fileThree)), "", ""), nil, opts)
 
 	test := func(token string) (int, []byte) {
 		rec := httptest.NewRecorder()
@@ -1177,7 +1177,7 @@ func testWebPresignedGetHandler(obj ObjectLayer, instanceType string, t TestErrH
 
 	data := bytes.Repeat([]byte("a"), objectSize)
 	metadata := map[string]string{"etag": "c9a34cfc85d982698c6ac89f76071abd"}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), metadata["etag"], ""), metadata, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Was not able to upload an object, %v", err)
 	}

cmd/xl-sets.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -598,7 +597,7 @@ func (s *xlSets) GetObject(ctx context.Context, bucket, object string, startOffs
 }
 
 // PutObject - writes an object to hashedSet based on the object name.
-func (s *xlSets) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+func (s *xlSets) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return s.getHashedSet(object).PutObject(ctx, bucket, object, data, metadata, opts)
 }
 
@@ -630,8 +629,7 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 		}
 		defer objectDWLock.Unlock()
 	}
-
-	return destSet.putObject(ctx, destBucket, destObject, srcInfo.Reader, srcInfo.UserDefined, dstOpts)
+	return destSet.putObject(ctx, destBucket, destObject, srcInfo.PutObjReader, srcInfo.UserDefined, dstOpts)
 }
 
 // Returns function "listDir" of the type listDirFunc.
@@ -828,11 +826,11 @@ func (s *xlSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destB
 	startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (partInfo PartInfo, err error) {
 	destSet := s.getHashedSet(destObject)
 
-	return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, srcInfo.Reader, dstOpts)
+	return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, NewPutObjReader(srcInfo.Reader, nil, nil), dstOpts)
 }
 
 // PutObjectPart - writes part of an object to hashedSet based on the object name.
-func (s *xlSets) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
+func (s *xlSets) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
 	return s.getHashedSet(object).PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 }
 
@@ -847,8 +845,8 @@ func (s *xlSets) AbortMultipartUpload(ctx context.Context, bucket, object, uploa
 }
 
 // CompleteMultipartUpload - completes a pending multipart transaction, on hashedSet based on object name.
-func (s *xlSets) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
-	return s.getHashedSet(object).CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts)
+func (s *xlSets) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	return s.getHashedSet(object).CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 }
 
 /*

cmd/xl-v1-common_test.go

@@ -43,7 +43,7 @@ func TestXLParentDirIsObject(t *testing.T) {
 	}
 	objectContent := "12345"
 	objInfo, err := obj.PutObject(context.Background(), bucketName, objectName,
-		mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
+		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/xl-v1-healing-common_test.go

@@ -200,7 +200,7 @@ func TestListOnlineDisks(t *testing.T) {
 			t.Fatalf("Failed to make a bucket %v", err)
 		}
 
-		_, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{})
 		if err != nil {
 			t.Fatalf("Failed to putObject %v", err)
 		}
@@ -292,7 +292,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
 
-	_, err = obj.PutObject(ctx, bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}

cmd/xl-v1-healing.go

@@ -636,9 +636,9 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 	reqInfo := logger.GetReqInfo(ctx)
 	var newReqInfo *logger.ReqInfo
 	if reqInfo != nil {
-		newReqInfo = logger.NewReqInfo(reqInfo.RemoteHost, reqInfo.UserAgent, reqInfo.RequestID, reqInfo.API, bucket, object)
+		newReqInfo = logger.NewReqInfo(reqInfo.RemoteHost, reqInfo.UserAgent, reqInfo.DeploymentID, reqInfo.RequestID, reqInfo.API, bucket, object)
 	} else {
-		newReqInfo = logger.NewReqInfo("", "", "", "Heal", bucket, object)
+		newReqInfo = logger.NewReqInfo("", "", globalDeploymentID, "", "Heal", bucket, object)
 	}
 	healCtx := logger.SetReqInfo(context.Background(), newReqInfo)