Update KMS readme with vault quick start guide (#6747)

This commit adds key-rotation for SSE-S3 objects.
To execute a key-rotation a SSE-S3 client must
 - specify the `X-Amz-Server-Side-Encryption: AES256` header
   for the destination
 - The source == destination for the COPY operation.

Fixes #6754

.travis.yml

@@ -1,10 +1,4 @@
 go_import_path: github.com/minio/minio
-sudo: required
-
-services:
- - docker
-
-dist: trusty
 
 language: go
 
@@ -18,9 +12,11 @@ branches:
 matrix:
   include:
     - os: linux
+      dist: trusty
+      sudo: required
       env:
         - ARCH=x86_64
-      go: 1.10.3
+      go: 1.10.4
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
@@ -29,9 +25,17 @@ matrix:
         - make verify
         - make coverage
         - cd browser && yarn && yarn test && cd ..
+    - os: windows
+      env:
+        - ARCH=x86_64
+      go: 1.10.4
+      script:
+        - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
+        - for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
+        - bash buildscripts/go-coverage.sh
 
 before_install:
-- nvm install stable
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
 
 after_success:
 - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -13,7 +13,7 @@ checks:
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 getdeps:
-	@echo "Installing golint" && go get -u github.com/golang/lint/golint
+	@echo "Installing golint" && go get -u golang.org/x/lint/golint
 	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
 	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
 	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell

README.md

@@ -17,7 +17,7 @@ docker run -p 9000:9000 minio/minio server /data
 docker pull minio/minio:edge
 docker run -p 9000:9000 minio/minio:edge server /data
 ```
-Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
+Note: Docker will not display the autogenerated keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use autogenerated keys with containers. Please visit Minio Docker quickstart guide for more information [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
 ### Homebrew

appveyor.yml

@@ -1,55 +0,0 @@
-# version format
-version: "{build}"
-
-# Operating system (build VM template)
-os: Windows Server 2012 R2
-
-# Platform.
-platform: x64
-
-clone_folder: c:\gopath\src\github.com\minio\minio
-
-# Environment variables
-environment:
-  GOPATH: c:\gopath
-  GOROOT: c:\go110
-
-# scripts that run after cloning repository
-install:
-  - set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
-  - go version
-  - go env
-  - python --version
-
-# To run your custom scripts instead of automatic MSBuild
-build_script:
-  # Compile
-  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
-  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
-  - appveyor AddCompilationMessage "Starting Compile"
-  - cd c:\gopath\src\github.com\minio\minio
-  - go run buildscripts/gen-ldflags.go > temp.txt
-  - set /p BUILD_LDFLAGS=<temp.txt
-  - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
-  - appveyor AddCompilationMessage "Compile Success"
-
-# To run your custom scripts instead of automatic tests
-test_script:
-  # Unit tests
-  - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
-  - mkdir build\coverage
-  - for /f "" %%G in ('go list github.com/minio/minio/... ^| find /i /v "browser/"') do ( go test -v -timeout 20m -race %%G )
-  - go test -v -timeout 20m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
-  - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
-
-after_test:
-  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
-  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
-  - ps: Push-AppveyorArtifact build\coverage\coverage.html
-  # Upload coverage report.
-  - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
-  - pip install codecov
-  - codecov -X gcov -f "build\coverage\coverage.txt"
-
-# to disable deployment
-deploy: off

buildscripts/go-coverage.sh

@@ -4,7 +4,7 @@ set -e
 echo "" > coverage.txt
 
 for d in $(go list ./... | grep -v browser); do
-    go test -coverprofile=profile.out -covermode=atomic "$d"
+    go test -v -coverprofile=profile.out -covermode=atomic "$d"
     if [ -f profile.out ]; then
         cat profile.out >> coverage.txt
         rm profile.out

cmd/acl-handlers.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -56,6 +57,8 @@ type accessControlPolicy struct {
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketACL")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -103,6 +106,8 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectACL")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]

cmd/admin-handlers.go

@@ -36,6 +36,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/quick"
 	"github.com/tidwall/gjson"
@@ -43,7 +44,7 @@ import (
 )
 
 const (
-	maxConfigJSONSize = 256 * 1024 // 256KiB
+	maxEConfigJSONSize = 262272
 )
 
 // Type-safe query params.
@@ -52,9 +53,10 @@ type mgmtQueryKey string
 // Only valid query params for mgmt admin APIs.
 const (
 	mgmtBucket      mgmtQueryKey = "bucket"
-	mgmtPrefix      mgmtQueryKey = "prefix"
-	mgmtClientToken mgmtQueryKey = "clientToken"
-	mgmtForceStart  mgmtQueryKey = "forceStart"
+	mgmtPrefix                   = "prefix"
+	mgmtClientToken              = "clientToken"
+	mgmtForceStart               = "forceStart"
+	mgmtForceStop                = "forceStop"
 )
 
 var (
@@ -105,7 +107,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 	// of read-quorum availability.
 	uptime, err := getPeerUptimes(globalAdminPeers)
 	if err != nil {
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		logger.LogIf(context.Background(), err)
 		return
 	}
@@ -401,7 +403,7 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 
 // extractHealInitParams - Validates params for heal init API.
 func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
-	hs madmin.HealOpts, clientToken string, forceStart bool,
+	hs madmin.HealOpts, clientToken string, forceStart bool, forceStop bool,
 	err APIErrorCode) {
 
 	vars := mux.Vars(r)
@@ -432,7 +434,9 @@ func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 	if _, ok := qParms[string(mgmtForceStart)]; ok {
 		forceStart = true
 	}
-
+	if _, ok := qParms[string(mgmtForceStop)]; ok {
+		forceStop = true
+	}
 	// ignore body if clientToken is provided
 	if clientToken == "" {
 		jerr := json.NewDecoder(r.Body).Decode(&hs)
@@ -483,7 +487,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bucket, objPrefix, hs, clientToken, forceStart, apiErr := extractHealInitParams(r)
+	bucket, objPrefix, hs, clientToken, forceStart, forceStop, apiErr := extractHealInitParams(r)
 	if apiErr != ErrNone {
 		writeErrorResponseJSON(w, apiErr, r.URL)
 		return
@@ -517,13 +521,35 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 				w.Write([]byte("\n\r"))
 				w.(http.Flusher).Flush()
 			case hr := <-respCh:
-				switch {
-				case hr.errCode == ErrNone:
-					writeSuccessResponseJSON(w, hr.respBytes)
-				case hr.errBody == "":
-					writeErrorResponseJSON(w, hr.errCode, r.URL)
+				switch hr.errCode {
+				case ErrNone:
+					if started {
+						w.Write(hr.respBytes)
+						w.(http.Flusher).Flush()
+					} else {
+						writeSuccessResponseJSON(w, hr.respBytes)
+					}
 				default:
-					writeCustomErrorResponseJSON(w, hr.errCode, hr.errBody, r.URL)
+					apiError := getAPIError(hr.errCode)
+					var errorRespJSON []byte
+					if hr.errBody == "" {
+						errorRespJSON = encodeResponseJSON(getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey)))
+					} else {
+						errorRespJSON = encodeResponseJSON(APIErrorResponse{
+							Code:      apiError.Code,
+							Message:   hr.errBody,
+							Resource:  r.URL.Path,
+							RequestID: w.Header().Get(responseRequestIDKey),
+							HostID:    "3L137",
+						})
+					}
+					if !started {
+						setCommonHeaders(w)
+						w.Header().Set("Content-Type", string(mimeJSON))
+						w.WriteHeader(apiError.HTTPStatusCode)
+					}
+					w.Write(errorRespJSON)
+					w.(http.Flusher).Flush()
 				}
 				break forLoop
 			}
@@ -534,34 +560,61 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	info := objLayer.StorageInfo(ctx)
 	numDisks := info.Backend.OfflineDisks + info.Backend.OnlineDisks
 
-	if clientToken == "" {
-		// Not a status request
-		nh := newHealSequence(bucket, objPrefix, handlers.GetSourceIP(r),
-			numDisks, hs, forceStart)
+	healPath := pathJoin(bucket, objPrefix)
+	if clientToken == "" && !forceStart && !forceStop {
+		nh, exists := globalAllHealState.getHealSequence(healPath)
+		if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
+			b, err := json.Marshal(madmin.HealStartSuccess{
+				ClientToken:   nh.clientToken,
+				ClientAddress: nh.clientAddress,
+				StartTime:     nh.startTime,
+			})
+			if err != nil {
+				logger.LogIf(context.Background(), err)
+				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+				return
+			}
+			// Client token not specified but a heal sequence exists on a path,
+			// Send the token back to client.
+			writeSuccessResponseJSON(w, b)
+			return
+		}
+	}
 
-		respCh := make(chan healResp)
-		go func() {
-			respBytes, errCode, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
-			hr := healResp{respBytes, errCode, errMsg}
-			respCh <- hr
-		}()
-
-		// Due to the force-starting functionality, the Launch
-		// call above can take a long time - to keep the
-		// connection alive, we start sending whitespace
-		keepConnLive(w, respCh)
-	} else {
+	if clientToken != "" && !forceStart && !forceStop {
 		// Since clientToken is given, fetch heal status from running
 		// heal sequence.
-		path := bucket + "/" + objPrefix
 		respBytes, errCode := globalAllHealState.PopHealStatusJSON(
-			path, clientToken)
+			healPath, clientToken)
 		if errCode != ErrNone {
 			writeErrorResponseJSON(w, errCode, r.URL)
 		} else {
 			writeSuccessResponseJSON(w, respBytes)
 		}
+		return
 	}
+
+	respCh := make(chan healResp)
+	switch {
+	case forceStop:
+		go func() {
+			respBytes, errCode := globalAllHealState.stopHealSequence(healPath)
+			hr := healResp{respBytes: respBytes, errCode: errCode}
+			respCh <- hr
+		}()
+	case clientToken == "":
+		nh := newHealSequence(bucket, objPrefix, handlers.GetSourceIP(r), numDisks, hs, forceStart)
+		go func() {
+			respBytes, errCode, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
+			hr := healResp{respBytes, errCode, errMsg}
+			respCh <- hr
+		}()
+	}
+
+	// Due to the force-starting functionality, the Launch
+	// call above can take a long time - to keep the
+	// connection alive, we start sending whitespace
+	keepConnLive(w, respCh)
 }
 
 // GetConfigHandler - GET /minio/admin/v1/config
@@ -597,7 +650,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	password := config.GetCredential().SecretKey
-	econfigData, err := madmin.EncryptServerConfigData(password, configData)
+	econfigData, err := madmin.EncryptData(password, configData)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
@@ -681,7 +734,7 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	password := config.GetCredential().SecretKey
-	econfigData, err := madmin.EncryptServerConfigData(password, []byte(newConfigStr))
+	econfigData, err := madmin.EncryptData(password, []byte(newConfigStr))
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
@@ -702,6 +755,348 @@ func toAdminAPIErrCode(err error) APIErrorCode {
 	}
 }
 
+// RemoveUser - DELETE /minio/admin/v1/remove-user?accessKey=<access_key>
+func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveUser")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+	}
+}
+
+// ListUsers - GET /minio/admin/v1/list-users
+func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListUsers")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	allCredentials, err := globalIAMSys.ListUsers()
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(allCredentials)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrInternalError, r.URL)
+		return
+	}
+
+	password := globalServerConfig.GetCredential().SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrInternalError, r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// SetUserStatus - PUT /minio/admin/v1/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
+func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetUserStatus")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+	status := vars["status"]
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalServerConfig.GetCredential().AccessKey {
+		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		return
+	}
+
+	if err := globalIAMSys.SetUserStatus(accessKey, madmin.AccountStatus(status)); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+}
+
+// AddUser - PUT /minio/admin/v1/add-user?accessKey=<access_key>
+func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddUser")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalServerConfig.GetCredential().AccessKey {
+		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
+		return
+	}
+
+	password := globalServerConfig.GetCredential().SecretKey
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
+		return
+	}
+
+	var uinfo madmin.UserInfo
+	if err = json.Unmarshal(configBytes, &uinfo); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
+		return
+	}
+
+	if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+}
+
+// ListCannedPolicies - GET /minio/admin/v1/list-canned-policies
+func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListCannedPolicies")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	policies, err := globalIAMSys.ListCannedPolicies()
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(policies); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// RemoveCannedPolicy - DELETE /minio/admin/v1/remove-canned-policy?name=<policy_name>
+func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveCannedPolicy")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["name"]
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	if err := globalIAMSys.DeleteCannedPolicy(policyName); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+}
+
+// AddCannedPolicy - PUT /minio/admin/v1/add-canned-policy?name=<policy_name>
+func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddCannedPolicy")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["name"]
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	// Error out if Content-Length is missing.
+	if r.ContentLength <= 0 {
+		writeErrorResponseJSON(w, ErrMissingContentLength, r.URL)
+		return
+	}
+
+	// Error out if Content-Length is beyond allowed size.
+	if r.ContentLength > maxBucketPolicySize {
+		writeErrorResponseJSON(w, ErrEntityTooLarge, r.URL)
+		return
+	}
+
+	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponseJSON(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	// Version in policy must not be empty
+	if iamPolicy.Version == "" {
+		writeErrorResponseJSON(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	if err = globalIAMSys.SetCannedPolicy(policyName, *iamPolicy); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+}
+
+// SetUserPolicy - PUT /minio/admin/v1/set-user-policy?accessKey=<access_key>&name=<policy_name>
+func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetUserPolicy")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+	policyName := vars["name"]
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalServerConfig.GetCredential().AccessKey {
+		writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
+		return
+	}
+
+	if err := globalIAMSys.SetUserPolicy(accessKey, policyName); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+	}
+}
+
 // SetConfigHandler - PUT /minio/admin/v1/config
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigHandler")
@@ -726,22 +1121,14 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	// Read configuration bytes from request body.
-	configBuf := make([]byte, maxConfigJSONSize+1)
-	n, err := io.ReadFull(r.Body, configBuf)
-	if err == nil {
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
 		return
 	}
-	if err != io.ErrUnexpectedEOF {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
-		return
-	}
 
 	password := globalServerConfig.GetCredential().SecretKey
-	configBytes, err := madmin.DecryptServerConfigData(password, bytes.NewReader(configBuf[:n]))
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
@@ -757,8 +1144,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	var config serverConfig
-	err = json.Unmarshal(configBytes, &config)
-	if err != nil {
+	if err = json.Unmarshal(configBytes, &config); err != nil {
 		logger.LogIf(ctx, err)
 		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
 		return
@@ -780,6 +1166,11 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	if err = config.TestNotificationTargets(); err != nil {
+		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
+		return
+	}
+
 	if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return
@@ -860,7 +1251,7 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 			writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
 			return
 		}
-		elem, dErr := madmin.DecryptServerConfigData(password, bytes.NewBuffer([]byte(encryptedElem)))
+		elem, dErr := madmin.DecryptData(password, bytes.NewBuffer([]byte(encryptedElem)))
 		if dErr != nil {
 			logger.LogIf(ctx, dErr)
 			writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
@@ -920,11 +1311,10 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// UpdateCredsHandler - POST /minio/admin/v1/config/credential
+// UpdateAdminCredsHandler - POST /minio/admin/v1/config/credential
 // ----------
-// Update credentials in a minio server. In a distributed setup,
-// update all the servers in the cluster.
-func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
+// Update admin credentials in a minio server
+func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 	r *http.Request) {
 
 	ctx := newContext(r, w, "UpdateCredentialsHandler")
@@ -938,7 +1328,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 
 	// Avoid setting new credentials when they are already passed
 	// by the environment. Deny if WORM is enabled.
-	if globalIsEnvCreds {
+	if globalIsEnvCreds || globalWORMEnabled {
 		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
 		return
 	}
@@ -950,22 +1340,14 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Read configuration bytes from request body.
-	configBuf := make([]byte, maxConfigJSONSize+1)
-	n, err := io.ReadFull(r.Body, configBuf)
-	if err == nil {
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
 		return
 	}
-	if err != io.ErrUnexpectedEOF {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
-		return
-	}
 
 	password := globalServerConfig.GetCredential().SecretKey
-	configBytes, err := madmin.DecryptServerConfigData(password, bytes.NewReader(configBuf[:n]))
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
@@ -982,7 +1364,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 
 	creds, err := auth.CreateCredentials(req.AccessKey, req.SecretKey)
 	if err != nil {
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return
 	}
 
@@ -993,6 +1375,9 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 	// Update local credentials in memory.
 	globalServerConfig.SetCredential(creds)
 
+	// Set active creds.
+	globalActiveCred = creds
+
 	if err = saveServerConfig(ctx, objectAPI, globalServerConfig); err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return

cmd/admin-handlers_test.go

@@ -38,172 +38,184 @@ import (
 
 var (
 	configJSON = []byte(`{
-	"version": "30",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "",
-	"worm": "off",
-	"storageclass": {
-		"standard": "",
-		"rrs": ""
-	},
-	"cache": {
-		"drives": [],
-		"expiry": 90,
-		"maxuse": 80,
-		"exclude": []
-	},
-	"kms": {
-		"vault": {
-			"endpoint": "",
-			"auth": {
-				"type": "",
-				"approle": {
-					"id": "",
-					"secret": ""
-				}
-			},
-			"key-id": {
-				"name": "",
-				"version": 0
-			}
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"deliveryMode": 0,
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"url": "",
-				"index": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": "",
-				"tls" : {
-					"enable" : false,
-					"skipVerify" : false,
-					"clientAuth" : 0
-				},
-				"sasl" : {
-					"enable" : false,
-					"username" : "",
-					"password" : ""
-				}
-			}
-		},
-		"mqtt": {
-			"1": {
-				"enable": false,
-				"broker": "",
-				"topic": "",
-				"qos": 0,
-				"clientId": "",
-				"username": "",
-				"password": "",
-				"reconnectInterval": 0,
-				"keepAliveInterval": 0
-			}
-		},
-		"mysql": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"dsnString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	},
-	"logger": {
-		"console": {
-			"enabled": true
-		},
-		"http": {
-			"target1": {
-				"enabled": false,
-				"endpoint": "https://username:password@example.com/api"
-			}
-		}
-	},
-	"compress": {
-		"enabled": false,
-        	"extensions":[".txt",".log",".csv",".json"],
-        	"mime-types":["text/csv","text/plain","application/json"]
-	}
-}`)
+  "version": "31",
+  "credential": {
+    "accessKey": "minio",
+    "secretKey": "minio123"
+  },
+  "region": "us-east-1",
+  "worm": "off",
+  "storageclass": {
+    "standard": "",
+    "rrs": ""
+  },
+  "cache": {
+    "drives": [],
+    "expiry": 90,
+    "maxuse": 80,
+    "exclude": []
+  },
+  "kms": {
+    "vault": {
+      "endpoint": "",
+      "auth": {
+        "type": "",
+        "approle": {
+          "id": "",
+          "secret": ""
+        }
+      },
+      "key-id": {
+        "name": "",
+        "version": 0
+      }
+    }
+  },
+  "notify": {
+    "amqp": {
+      "1": {
+        "enable": false,
+        "url": "",
+        "exchange": "",
+        "routingKey": "",
+        "exchangeType": "",
+        "deliveryMode": 0,
+        "mandatory": false,
+        "immediate": false,
+        "durable": false,
+        "internal": false,
+        "noWait": false,
+        "autoDeleted": false
+      }
+    },
+    "elasticsearch": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "url": "",
+        "index": ""
+      }
+    },
+    "kafka": {
+      "1": {
+        "enable": false,
+        "brokers": null,
+        "topic": "",
+        "tls": {
+          "enable": false,
+          "skipVerify": false,
+          "clientAuth": 0
+        },
+        "sasl": {
+          "enable": false,
+          "username": "",
+          "password": ""
+        }
+      }
+    },
+    "mqtt": {
+      "1": {
+        "enable": false,
+        "broker": "",
+        "topic": "",
+        "qos": 0,
+        "clientId": "",
+        "username": "",
+        "password": "",
+        "reconnectInterval": 0,
+        "keepAliveInterval": 0
+      }
+    },
+    "mysql": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "dsnString": "",
+        "table": "",
+        "host": "",
+        "port": "",
+        "user": "",
+        "password": "",
+        "database": ""
+      }
+    },
+    "nats": {
+      "1": {
+        "enable": false,
+        "address": "",
+        "subject": "",
+        "username": "",
+        "password": "",
+        "token": "",
+        "secure": false,
+        "pingInterval": 0,
+        "streaming": {
+          "enable": false,
+          "clusterID": "",
+          "clientID": "",
+          "async": false,
+          "maxPubAcksInflight": 0
+        }
+      }
+    },
+    "postgresql": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "connectionString": "",
+        "table": "",
+        "host": "",
+        "port": "",
+        "user": "",
+        "password": "",
+        "database": ""
+      }
+    },
+    "redis": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "address": "",
+        "password": "",
+        "key": ""
+      }
+    },
+    "webhook": {
+      "1": {
+        "enable": false,
+        "endpoint": ""
+      }
+    }
+  },
+  "logger": {
+    "console": {
+      "enabled": true
+    },
+    "http": {
+      "1": {
+        "enabled": false,
+        "endpoint": "https://username:password@example.com/api"
+      }
+    }
+  },
+  "compress": {
+    "enabled": false,
+    "extensions":[".txt",".log",".csv",".json"],
+    "mime-types":["text/csv","text/plain","application/json"]
+  },
+  "openid": {
+    "jwks": {
+      "url": ""
+    }
+  },
+  "policy": {
+    "opa": {
+      "url": "",
+      "authToken": ""
+    }
+  }
+}
+`)
 )
 
 // adminXLTestBed - encapsulates subsystems that need to be setup for
@@ -485,6 +497,8 @@ func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*htt
 
 	// Set body
 	req.Body = ioutil.NopCloser(bytes.NewReader(body))
+	req.ContentLength = int64(len(body))
+
 	// Set sha-sum header
 	req.Header.Set("X-Amz-Content-Sha256", getSHA256Hash(body))
 
@@ -615,7 +629,7 @@ func TestServiceSetCreds(t *testing.T) {
 			t.Fatalf("JSONify err: %v", err)
 		}
 
-		ebody, err := madmin.EncryptServerConfigData(credentials.SecretKey, body)
+		ebody, err := madmin.EncryptData(credentials.SecretKey, body)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -718,7 +732,7 @@ func TestSetConfigHandler(t *testing.T) {
 	queryVal.Set("config", "")
 
 	password := globalServerConfig.GetCredential().SecretKey
-	econfigJSON, err := madmin.EncryptServerConfigData(password, configJSON)
+	econfigJSON, err := madmin.EncryptData(password, configJSON)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -738,7 +752,7 @@ func TestSetConfigHandler(t *testing.T) {
 	// Check that a very large config file returns an error.
 	{
 		// Make a large enough config string
-		invalidCfg := []byte(strings.Repeat("A", maxConfigJSONSize+1))
+		invalidCfg := []byte(strings.Repeat("A", maxEConfigJSONSize+1))
 		req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
 			int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
 		if err != nil {
@@ -768,7 +782,7 @@ func TestSetConfigHandler(t *testing.T) {
 		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
-			!strings.Contains(respBody, "JSON configuration provided has objects with duplicate keys") {
+			!strings.Contains(respBody, "JSON configuration provided is of incorrect format") {
 			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
 		}
 	}

cmd/admin-heal-ops.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"runtime"
 	"strings"
 	"sync"
@@ -123,6 +124,35 @@ func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists b
 	return h, exists
 }
 
+func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
+	var hsp madmin.HealStopSuccess
+	he, exists := ahs.getHealSequence(path)
+	if !exists {
+		hsp = madmin.HealStopSuccess{
+			ClientToken: "invalid",
+			StartTime:   UTCNow(),
+		}
+	} else {
+		hsp = madmin.HealStopSuccess{
+			ClientToken:   he.clientToken,
+			ClientAddress: he.clientAddress,
+			StartTime:     he.startTime,
+		}
+
+		he.stop()
+		for !he.hasEnded() {
+			time.Sleep(1 * time.Second)
+		}
+		ahs.Lock()
+		defer ahs.Unlock()
+		// Heal sequence explicitly stopped, remove it.
+		delete(ahs.healSeqMap, path)
+	}
+
+	b, err := json.Marshal(&hsp)
+	return b, toAdminAPIErrCode(err)
+}
+
 // LaunchNewHealSequence - launches a background routine that performs
 // healing according to the healSequence argument. For each heal
 // sequence, state is stored in the `globalAllHealState`, which is a
@@ -143,20 +173,20 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 			existsAndLive = true
 		}
 	}
+
 	if existsAndLive {
 		// A heal sequence exists on the given path.
 		if h.forceStarted {
-			// stop the running heal sequence - wait for
-			// it to finish.
+			// stop the running heal sequence - wait for it to finish.
 			he.stop()
 			for !he.hasEnded() {
-				time.Sleep(10 * time.Second)
+				time.Sleep(1 * time.Second)
 			}
 		} else {
 			errMsg = "Heal is already running on the given path " +
 				"(use force-start option to stop and start afresh). " +
-				fmt.Sprintf("The heal was started by IP %s at %s",
-					h.clientAddress, h.startTime)
+				fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
+					h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken)
 
 			return nil, ErrHealAlreadyRunning, errMsg
 		}
@@ -285,7 +315,7 @@ type healSequence struct {
 	// bucket, and prefix on which heal seq. was initiated
 	bucket, objPrefix string
 
-	// path is just bucket + "/" + objPrefix
+	// path is just pathJoin(bucket, objPrefix)
 	path string
 
 	// time at which heal sequence was started
@@ -330,7 +360,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 	return &healSequence{
 		bucket:        bucket,
 		objPrefix:     objPrefix,
-		path:          bucket + "/" + objPrefix,
+		path:          pathJoin(bucket, objPrefix),
 		startTime:     UTCNow(),
 		clientToken:   mustGetUUID(),
 		clientAddress: clientAddr,
@@ -552,7 +582,7 @@ func (h *healSequence) healConfig() error {
 			// before proceeding to heal
 			waitCount := 60
 			// Any requests in progress, delay the heal.
-			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
 				waitCount--
 				time.Sleep(1 * time.Second)
 			}
@@ -698,7 +728,7 @@ func (h *healSequence) healBucket(bucket string) error {
 			// before proceeding to heal
 			waitCount := 60
 			// Any requests in progress, delay the heal.
-			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
 				waitCount--
 				time.Sleep(1 * time.Second)
 			}

cmd/admin-router.go

@@ -53,12 +53,14 @@ func registerAdminRouter(router *mux.Router) {
 	// Info operations
 	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 
-	/// Heal operations
+	if globalIsDistXL || globalIsXL {
+		/// Heal operations
 
-	// Heal processing endpoint.
-	adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		// Heal processing endpoint.
+		adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+	}
 
 	// Profiling operations
 	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
@@ -68,7 +70,7 @@ func registerAdminRouter(router *mux.Router) {
 	/// Config operations
 
 	// Update credentials
-	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateCredentialsHandler))
+	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
 	// Get config
 	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 	// Set config
@@ -78,4 +80,31 @@ func registerAdminRouter(router *mux.Router) {
 	adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
 	// Set config keys/values
 	adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
+
+	// -- IAM APIs --
+
+	// Add policy IAM
+	adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+
+	// Add user IAM
+	adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+	adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
+		Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
+	adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
+		Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+
+	// Remove policy IAM
+	adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+
+	// Remove user IAM
+	adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+
+	// List users
+	adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+
+	// List policies
+	adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+
+	// If none of the routes match.
+	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandler))
 }

cmd/api-datatypes.go

@@ -25,90 +25,6 @@ const (
 	responseRequestIDKey = "x-amz-request-id"
 )
 
-// CSVFileHeaderInfo -Can be either USE IGNORE OR NONE, defines what to do with
-// the first row
-type CSVFileHeaderInfo string
-
-// Constants for file header info.
-const (
-	CSVFileHeaderInfoNone   CSVFileHeaderInfo = "NONE"
-	CSVFileHeaderInfoIgnore                   = "IGNORE"
-	CSVFileHeaderInfoUse                      = "USE"
-)
-
-// SelectCompressionType - ONLY GZIP is supported
-type SelectCompressionType string
-
-// Constants for compression types under select API.
-const (
-	SelectCompressionNONE SelectCompressionType = "NONE"
-	SelectCompressionGZIP                       = "GZIP"
-	SelectCompressionBZIP                       = "BZIP2"
-)
-
-// CSVQuoteFields - Can be either Always or AsNeeded
-type CSVQuoteFields string
-
-// Constants for csv quote styles.
-const (
-	CSVQuoteFieldsAlways   CSVQuoteFields = "Always"
-	CSVQuoteFieldsAsNeeded                = "AsNeeded"
-)
-
-// QueryExpressionType - Currently can only be SQL
-type QueryExpressionType string
-
-// Constants for expression type.
-const (
-	QueryExpressionTypeSQL QueryExpressionType = "SQL"
-)
-
-// JSONType determines json input serialization type.
-type JSONType string
-
-// Constants for JSONTypes.
-const (
-	JSONDocumentType JSONType = "Document"
-	JSONLinesType             = "Lines"
-)
-
-// ObjectSelectRequest - represents the input select body
-type ObjectSelectRequest struct {
-	XMLName            xml.Name `xml:"SelectObjectContentRequest" json:"-"`
-	Expression         string
-	ExpressionType     QueryExpressionType
-	InputSerialization struct {
-		CompressionType SelectCompressionType
-		Parquet         *struct{}
-		CSV             *struct {
-			FileHeaderInfo       CSVFileHeaderInfo
-			RecordDelimiter      string
-			FieldDelimiter       string
-			QuoteCharacter       string
-			QuoteEscapeCharacter string
-			Comments             string
-		}
-		JSON *struct {
-			Type JSONType
-		}
-	}
-	OutputSerialization struct {
-		CSV *struct {
-			QuoteFields          CSVQuoteFields
-			RecordDelimiter      string
-			FieldDelimiter       string
-			QuoteCharacter       string
-			QuoteEscapeCharacter string
-		}
-		JSON *struct {
-			RecordDelimiter string
-		}
-	}
-	RequestProgress struct {
-		Enabled bool
-	}
-}
-
 // ObjectIdentifier carries key name for the object to delete.
 type ObjectIdentifier struct {
 	ObjectName string `xml:"Key"`

cmd/api-errors.go

@@ -187,11 +187,15 @@ const (
 	// new error codes here.
 
 	ErrMalformedJSON
+	ErrAdminNoSuchUser
+	ErrAdminNoSuchPolicy
+	ErrAdminInvalidArgument
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
 	ErrAdminConfigNoQuorum
 	ErrAdminConfigTooLarge
 	ErrAdminConfigBadJSON
+	ErrAdminConfigDuplicateKeys
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
@@ -249,7 +253,6 @@ const (
 	ErrParseUnsupportedAlias
 	ErrParseUnsupportedSyntax
 	ErrParseUnknownOperator
-	ErrParseInvalidPathComponent
 	ErrParseMissingIdentAfterAt
 	ErrParseUnexpectedOperator
 	ErrParseUnexpectedTerm
@@ -288,7 +291,6 @@ const (
 	ErrEvaluatorInvalidTimestampFormatPatternToken
 	ErrEvaluatorInvalidTimestampFormatPatternSymbol
 	ErrEvaluatorBindingDoesNotExist
-	ErrInvalidColumnIndex
 	ErrMissingHeaders
 	ErrAdminConfigNotificationTargetsFailed
 	ErrAdminProfilerNotEnabled
@@ -863,6 +865,21 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The JSON you provided was not well-formed or did not validate against our published format.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminNoSuchUser: {
+		Code:           "XMinioAdminNoSuchUser",
+		Description:    "The specified user does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminNoSuchPolicy: {
+		Code:           "XMinioAdminNoSuchPolicy",
+		Description:    "The canned policy does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminInvalidArgument: {
+		Code:           "XMinioAdminInvalidArgument",
+		Description:    "Invalid arguments specified.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminInvalidAccessKey: {
 		Code:           "XMinioAdminInvalidAccessKey",
 		Description:    "The access key is invalid.",
@@ -881,11 +898,16 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	ErrAdminConfigTooLarge: {
 		Code: "XMinioAdminConfigTooLarge",
 		Description: fmt.Sprintf("Configuration data provided exceeds the allowed maximum of %d bytes",
-			maxConfigJSONSize),
+			maxEConfigJSONSize),
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrAdminConfigBadJSON: {
 		Code:           "XMinioAdminConfigBadJSON",
+		Description:    "JSON configuration provided is of incorrect format",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminConfigDuplicateKeys: {
+		Code:           "XMinioAdminConfigDuplicateKeys",
 		Description:    "JSON configuration provided has objects with duplicate keys",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
@@ -1199,11 +1221,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The SQL expression contains an invalid operator.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrParseInvalidPathComponent: {
-		Code:           "ParseInvalidPathComponent",
-		Description:    "The SQL expression contains an invalid path component.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrParseMissingIdentAfterAt: {
 		Code:           "ParseMissingIdentAfterAt",
 		Description:    "Did not find the expected identifier after the @ symbol in the SQL expression.",
@@ -1389,11 +1406,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Time stamp format pattern contains an invalid symbol in the SQL expression.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrInvalidColumnIndex: {
-		Code:           "InvalidColumnIndex",
-		Description:    "Column index in the SQL expression is invalid.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrEvaluatorBindingDoesNotExist: {
 		Code:           "ErrEvaluatorBindingDoesNotExist",
 		Description:    "A column name or a path provided does not exist in the SQL expression",
@@ -1422,6 +1434,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 	// Verify if the underlying error is signature mismatch.
 	switch err {
+	case errInvalidArgument:
+		apiErr = ErrAdminInvalidArgument
+	case errNoSuchUser:
+		apiErr = ErrAdminNoSuchUser
+	case errNoSuchPolicy:
+		apiErr = ErrAdminNoSuchPolicy
 	case errSignatureMismatch:
 		apiErr = ErrSignatureDoesNotMatch
 	case errInvalidRange:
@@ -1435,10 +1453,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
 	// SSE errors
+	case errInvalidEncryptionParameters:
+		apiErr = ErrInvalidEncryptionParameters
 	case crypto.ErrInvalidEncryptionMethod:
 		apiErr = ErrInvalidEncryptionMethod
-	case errInsecureSSERequest:
-		apiErr = ErrInsecureSSECustomerRequest
 	case crypto.ErrInvalidCustomerAlgorithm:
 		apiErr = ErrInvalidSSECustomerAlgorithm
 	case crypto.ErrInvalidCustomerKey:
@@ -1547,8 +1565,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrParseUnsupportedSyntax
 	case s3select.ErrParseUnknownOperator:
 		apiErr = ErrParseUnknownOperator
-	case s3select.ErrParseInvalidPathComponent:
-		apiErr = ErrParseInvalidPathComponent
 	case s3select.ErrParseMissingIdentAfterAt:
 		apiErr = ErrParseMissingIdentAfterAt
 	case s3select.ErrParseUnexpectedOperator:
@@ -1621,8 +1637,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEvaluatorInvalidTimestampFormatPatternToken
 	case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbol:
 		apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbol
-	case s3select.ErrInvalidColumnIndex:
-		apiErr = ErrInvalidColumnIndex
 	case s3select.ErrEvaluatorBindingDoesNotExist:
 		apiErr = ErrEvaluatorBindingDoesNotExist
 	case s3select.ErrMissingHeaders:

cmd/api-errors_test.go

@@ -52,7 +52,6 @@ var toAPIErrorCodeTests = []struct {
 	{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
 
 	// SSE-C errors
-	{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
 	{err: crypto.ErrInvalidCustomerAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
 	{err: crypto.ErrMissingCustomerKey, errCode: ErrMissingSSECustomerKey},
 	{err: crypto.ErrInvalidCustomerKey, errCode: ErrInvalidSSECustomerKey},

cmd/api-resources.go

@@ -22,15 +22,22 @@ import (
 )
 
 // Parse bucket url queries
-func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
-	prefix = values.Get("prefix")
-	marker = values.Get("marker")
-	delimiter = values.Get("delimiter")
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
 	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
 	} else {
 		maxkeys = maxObjectList
 	}
+
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	return
 }
@@ -47,44 +54,69 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 		}
 	}
 
+	if values.Get("max-keys") != "" {
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
+	} else {
+		maxkeys = maxObjectList
+	}
+
 	prefix = values.Get("prefix")
 	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
-	} else {
-		maxkeys = maxObjectList
-	}
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse bucket url queries for ?uploads
-func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	if values.Get("max-uploads") != "" {
+		var err error
+		if maxUploads, err = strconv.Atoi(values.Get("max-uploads")); err != nil {
+			errCode = ErrInvalidMaxUploads
+			return
+		}
+	} else {
+		maxUploads = maxUploadsList
+	}
+
 	prefix = values.Get("prefix")
 	keyMarker = values.Get("key-marker")
 	uploadIDMarker = values.Get("upload-id-marker")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-uploads") != "" {
-		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
-	} else {
-		maxUploads = maxUploadsList
-	}
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse object url queries
-func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
-	uploadID = values.Get("uploadId")
-	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string, errCode APIErrorCode) {
+	var err error
+	errCode = ErrNone
+
 	if values.Get("max-parts") != "" {
-		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+		if maxParts, err = strconv.Atoi(values.Get("max-parts")); err != nil {
+			errCode = ErrInvalidMaxParts
+			return
+		}
 	} else {
 		maxParts = maxPartsList
 	}
+
+	if values.Get("part-number-marker") != "" {
+		if partNumberMarker, err = strconv.Atoi(values.Get("part-number-marker")); err != nil {
+			errCode = ErrInvalidPartNumberMarker
+			return
+		}
+	}
+
+	uploadID = values.Get("uploadId")
 	encodingType = values.Get("encoding-type")
 	return
 }

cmd/api-resources_test.go

@@ -156,7 +156,10 @@ func TestListObjectsV1Resources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		prefix, marker, delimiter, maxKeys, encodingType := getListObjectsV1Args(testCase.values)
+		prefix, marker, delimiter, maxKeys, encodingType, argsErr := getListObjectsV1Args(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}
@@ -198,7 +201,10 @@ func TestGetObjectsResources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		uploadID, partNumberMarker, maxParts, encodingType := getObjectResources(testCase.values)
+		uploadID, partNumberMarker, maxParts, encodingType, argsErr := getObjectResources(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if uploadID != testCase.uploadID {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.uploadID, uploadID)
 		}

cmd/api-response.go

@@ -608,7 +608,7 @@ func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode,
 		Code:      apiError.Code,
 		Message:   errBody,
 		Resource:  reqURL.Path,
-		RequestID: "3L137",
+		RequestID: w.Header().Get(responseRequestIDKey),
 		HostID:    "3L137",
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)

cmd/auth-handler.go

@@ -27,8 +27,10 @@ import (
 	"net/http"
 	"strings"
 
+	jwtgo "github.com/dgrijalva/jwt-go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -86,6 +88,7 @@ const (
 	authTypeSigned
 	authTypeSignedV2
 	authTypeJWT
+	authTypeSTS
 )
 
 // Get request authentication type.
@@ -106,6 +109,8 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypePostPolicy
 	} else if _, ok := r.Header["Authorization"]; !ok {
 		return authTypeAnonymous
+	} else if _, ok := r.URL.Query()["Action"]; ok {
+		return authTypeSTS
 	}
 	return authTypeUnknown
 }
@@ -114,7 +119,21 @@ func getRequestAuthType(r *http.Request) authType {
 // It does not accept presigned or JWT or anonymous requests.
 func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
-	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok && getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) { // we only support V4 (no presign) with auth. body
+	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
+		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
+		// We only support admin credentials to access admin APIs.
+
+		var owner bool
+		_, owner, s3Err = getReqAccessKeyV4(r, region)
+		if s3Err != ErrNone {
+			return s3Err
+		}
+
+		if !owner {
+			return ErrAccessDenied
+		}
+
+		// we only support V4 (no presign) with auth body
 		s3Err = isReqAuthenticated(r, region)
 	}
 	if s3Err != ErrNone {
@@ -125,30 +144,62 @@ func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 	return s3Err
 }
 
-func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) APIErrorCode {
-	isOwner := true
-	accountName := globalServerConfig.GetCredential().AccessKey
+// Fetch the security token set by the client.
+func getSessionToken(r *http.Request) (token string) {
+	token = r.Header.Get("X-Amz-Security-Token")
+	if token != "" {
+		return token
+	}
+	return r.URL.Query().Get("X-Amz-Security-Token")
+}
 
+// Fetch claims in the security token returned by the client and validate the token.
+func getClaimsFromToken(r *http.Request) (map[string]interface{}, APIErrorCode) {
+	claims := make(map[string]interface{})
+	token := getSessionToken(r)
+	if token == "" {
+		return nil, ErrNone
+	}
+	p := &jwtgo.Parser{}
+	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
+	if err != nil {
+		return nil, toAPIErrorCode(errAuthentication)
+	}
+	if !jtoken.Valid {
+		return nil, toAPIErrorCode(errAuthentication)
+	}
+	return claims, ErrNone
+}
+
+// Check request auth type verifies the incoming http request
+// - validates the request signature
+// - validates the policy action if anonymous tests bucket policies if any,
+//   for authenticated requests validates IAM policies.
+// returns APIErrorCode if any to be replied to the client.
+func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
+	var accessKey string
+	var owner bool
 	switch getRequestAuthType(r) {
-	case authTypeUnknown:
+	case authTypeUnknown, authTypeStreamingSigned:
 		return ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
-		if errorCode := isReqAuthenticatedV2(r); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
+			return s3Err
 		}
+		accessKey, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
 		region := globalServerConfig.GetRegion()
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
 		}
-
-		if errorCode := isReqAuthenticated(r, region); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticated(r, region); s3Err != ErrNone {
+			return s3Err
 		}
-	default:
-		isOwner = false
-		accountName = ""
+		accessKey, owner, s3Err = getReqAccessKeyV4(r, region)
+	}
+	if s3Err != ErrNone {
+		return s3Err
 	}
 
 	// LocationConstraint is valid only for CreateBucketAction.
@@ -174,17 +225,36 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
 
-	if globalPolicySys.IsAllowed(policy.Args{
-		AccountName:     accountName,
-		Action:          action,
+	claims, s3Err := getClaimsFromToken(r)
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	if accessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     accessKey,
+			Action:          action,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, locationConstraint),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     accessKey,
+		Action:          iampolicy.Action(action),
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, locationConstraint),
-		IsOwner:         isOwner,
+		ConditionValues: getConditionValues(r, ""),
 		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
 	}) {
 		return ErrNone
 	}
-
 	return ErrAccessDenied
 }
 
@@ -288,7 +358,7 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	} else if aType == authTypeJWT {
 		// Validate Authorization header if its valid for JWT request.
-		if !isHTTPRequestValid(r) {
+		if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
 			w.WriteHeader(http.StatusUnauthorized)
 			return
 		}
@@ -297,3 +367,55 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
 }
+
+// isPutAllowed - check if PUT operation is allowed on the resource, this
+// call verifies bucket policies and IAM policies, supports multi user
+// checks etc.
+func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request) (s3Err APIErrorCode) {
+	var accessKey string
+	var owner bool
+	switch atype {
+	case authTypeUnknown:
+		return ErrAccessDenied
+	case authTypeSignedV2, authTypePresignedV2:
+		accessKey, owner, s3Err = getReqAccessKeyV2(r)
+	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
+		region := globalServerConfig.GetRegion()
+		accessKey, owner, s3Err = getReqAccessKeyV4(r, region)
+	}
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	claims, s3Err := getClaimsFromToken(r)
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	if accessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     accessKey,
+			Action:          policy.PutObjectAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     accessKey,
+		Action:          policy.PutObjectAction,
+		BucketName:      bucketName,
+		ConditionValues: getConditionValues(r, ""),
+		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
+	}) {
+		return ErrNone
+	}
+	return ErrAccessDenied
+}

cmd/bucket-handlers-listobjects.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/policy"
 )
@@ -58,6 +59,8 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -137,6 +140,8 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -152,7 +157,11 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	// Extract all the litsObjectsV1 query params to their native values.
-	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
+	prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 
 	// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
 	// does not throw an error.

cmd/bucket-handlers.go

@@ -89,6 +89,8 @@ func initFederatorBackend(objLayer ObjectLayer) {
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLocation")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -137,6 +139,8 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -151,7 +155,11 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 	if maxUploads < 0 {
 		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
 		return
@@ -184,6 +192,8 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBuckets")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -240,6 +250,8 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteMultipleObjects")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -370,10 +382,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			Object: ObjectInfo{
 				Name: dobj.ObjectName,
 			},
-			ReqParams: extractReqParams(r),
-			UserAgent: r.UserAgent(),
-			Host:      host,
-			Port:      port,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         host,
+			Port:         port,
 		})
 	}
 }
@@ -384,6 +397,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucket")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -460,6 +475,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PostPolicyBucket")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -640,13 +657,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedPost,
-		BucketName: objInfo.Bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedPost,
+		BucketName:   objInfo.Bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 
 	if successRedirect != "" {
@@ -682,6 +700,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadBucket")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -712,6 +732,8 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucket")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 

cmd/bucket-notification-handlers.go

@@ -44,6 +44,8 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketNotification")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
@@ -96,6 +98,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketNotification")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -139,7 +143,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err = saveNotificationConfig(objectAPI, bucketName, config); err != nil {
+	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -156,6 +160,8 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListenBucketNotification")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-policy-handlers.go

@@ -40,6 +40,8 @@ const (
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketPolicy")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -101,6 +103,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketPolicy")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -137,6 +141,8 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketPolicy")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)

cmd/certs.go

@@ -150,24 +150,20 @@ func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	return cert, nil
 }
 
-func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, c *certs.Certs, secureConn bool, err error) {
+func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return nil, nil, nil, false, nil
+		return nil, nil, false, nil
 	}
 
 	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return nil, nil, nil, false, err
+		return nil, nil, false, err
 	}
 
 	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
 	if err != nil {
-		return nil, nil, nil, false, err
-	}
-
-	if rootCAs, err = getRootCAs(getCADir()); err != nil {
-		return nil, nil, nil, false, err
+		return nil, nil, false, err
 	}
 
 	secureConn = true
-	return x509Certs, rootCAs, c, secureConn, nil
+	return x509Certs, c, secureConn, nil
 }

cmd/common-main.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"errors"
 	"net"
 	"os"
@@ -28,12 +29,12 @@ import (
 	etcd "github.com/coreos/etcd/clientv3"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
-
-	"github.com/minio/minio-go/pkg/set"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Check for updates and print a notification message
@@ -50,16 +51,30 @@ func checkUpdate(mode string) {
 
 // Load logger targets based on user's configuration
 func loadLoggers() {
+	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable audit HTTP logging through ENV.
+		logger.AddAuditTarget(logger.NewHTTP(auditEndpoint, NewCustomHTTPTransport()))
+	}
+
+	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable HTTP logging through ENV.
+		logger.AddTarget(logger.NewHTTP(loggerEndpoint, NewCustomHTTPTransport()))
+	} else {
+		for _, l := range globalServerConfig.Logger.HTTP {
+			if l.Enabled {
+				// Enable http logging
+				logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
+			}
+		}
+	}
+
 	if globalServerConfig.Logger.Console.Enabled {
 		// Enable console logging
 		logger.AddTarget(logger.NewConsole())
 	}
-	for _, l := range globalServerConfig.Logger.HTTP {
-		if l.Enabled {
-			// Enable http logging
-			logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
-		}
-	}
+
 }
 
 func handleCommonCmdArgs(ctx *cli.Context) {
@@ -122,6 +137,7 @@ func handleCommonEnvVars() {
 		if err != nil {
 			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
 		}
+		cred.Expiration = timeSentinel
 
 		// credential Envs are set globally.
 		globalIsEnvCreds = true
@@ -131,7 +147,7 @@ func handleCommonEnvVars() {
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
 		browserFlag, err := ParseBoolFlag(browser)
 		if err != nil {
-			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER environment variable")
+			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -150,19 +166,53 @@ func handleCommonEnvVars() {
 	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
 	if ok {
 		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+
+		var etcdSecure bool
+		for _, endpoint := range etcdEndpoints {
+			u, err := xnet.ParseURL(endpoint)
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+			}
+			// If one of the endpoint is https, we will use https directly.
+			etcdSecure = etcdSecure || u.Scheme == "https"
+		}
+
 		var err error
-		globalEtcdClient, err = etcd.New(etcd.Config{
-			Endpoints:         etcdEndpoints,
-			DialTimeout:       defaultDialTimeout,
-			DialKeepAliveTime: defaultDialKeepAlive,
-		})
+		if etcdSecure {
+			// This is only to support client side certificate authentication
+			// https://coreos.com/etcd/docs/latest/op-guide/security.html
+			etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT")
+			etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY")
+			var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+			if ok1 && ok2 {
+				getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+					cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
+					return &cert, terr
+				}
+			}
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+				TLS: &tls.Config{
+					RootCAs:              globalRootCAs,
+					GetClientCertificate: getClientCertificate,
+				},
+			})
+		} else {
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+			})
+		}
 		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
 	}
 
 	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
 	if globalDomainName != "" {
 		if _, ok = dns2.IsDomainName(globalDomainName); !ok {
-			logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN environment variable")
+			logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN value in environment variable")
 		}
 	}
 
@@ -257,7 +307,7 @@ func handleCommonEnvVars() {
 	if worm := os.Getenv("MINIO_WORM"); worm != "" {
 		wormFlag, err := ParseBoolFlag(worm)
 		if err != nil {
-			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Unable to validate MINIO_WORM environment variable")
+			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
 		}
 
 		// worm Envs are set globally, this does not represent

cmd/config-common.go

@@ -0,0 +1,133 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+)
+
+var errConfigNotFound = errors.New("config file not found")
+
+func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+	var buffer bytes.Buffer
+	// Read entire content by setting size to -1
+	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return nil, errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return nil, err
+	}
+
+	// Return config not found on empty content.
+	if buffer.Len() == 0 {
+		return nil, errConfigNotFound
+	}
+
+	return buffer.Bytes(), nil
+}
+
+func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	_, err := client.Delete(ctx, configFile)
+	return err
+}
+
+func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+}
+
+func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
+	_, err := client.Put(ctx, configFile, string(data))
+	return err
+}
+
+func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
+	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
+	if err != nil {
+		return err
+	}
+
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, hashReader, nil, ObjectOptions{})
+	return err
+}
+
+func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
+	resp, err := client.Get(ctx, configFile)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Count == 0 {
+		return nil, errConfigNotFound
+	}
+	for _, ev := range resp.Kvs {
+		if string(ev.Key) == configFile {
+			return ev.Value, nil
+		}
+	}
+	return nil, errConfigNotFound
+}
+
+// watchConfig - watches for changes on `configFile` on etcd and loads them.
+func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
+	if globalEtcdClient != nil {
+		for watchResp := range globalEtcdClient.Watch(context.Background(), configFile) {
+			for _, event := range watchResp.Events {
+				if event.IsModify() || event.IsCreate() {
+					loadCfgFn(objAPI)
+				}
+			}
+		}
+	}
+}
+
+func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	resp, err := globalEtcdClient.Get(ctx, configFile)
+	if err != nil {
+		return err
+	}
+	if resp.Count == 0 {
+		return errConfigNotFound
+	}
+	return nil
+}
+
+func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	if globalEtcdClient != nil {
+		return checkConfigEtcd(ctx, globalEtcdClient, configFile)
+	}
+
+	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return err
+	}
+	return nil
+}

cmd/config-current.go

@@ -20,15 +20,18 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"sync"
 
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
-
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Steps to move from version N to version N+1
@@ -40,9 +43,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "30"
+const serverConfigVersion = "31"
 
-type serverConfig = serverConfigV30
+type serverConfig = serverConfigV31
 
 var (
 	// globalServerConfig server config.
@@ -173,55 +176,55 @@ func (s *serverConfig) Validate() error {
 	// Worm, Cache and StorageClass values are already validated during json unmarshal
 	for _, v := range s.Notify.AMQP {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("amqp: %s", err.Error())
+			return fmt.Errorf("amqp: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Elasticsearch {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("elasticsearch: %s", err.Error())
+			return fmt.Errorf("elasticsearch: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Kafka {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("kafka: %s", err.Error())
+			return fmt.Errorf("kafka: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.MQTT {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("mqtt: %s", err.Error())
+			return fmt.Errorf("mqtt: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.MySQL {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("mysql: %s", err.Error())
+			return fmt.Errorf("mysql: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.NATS {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("nats: %s", err.Error())
+			return fmt.Errorf("nats: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.PostgreSQL {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("postgreSQL: %s", err.Error())
+			return fmt.Errorf("postgreSQL: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Redis {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("redis: %s", err.Error())
+			return fmt.Errorf("redis: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Webhook {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("webhook: %s", err.Error())
+			return fmt.Errorf("webhook: %s", err)
 		}
 	}
 
@@ -269,6 +272,20 @@ func (s *serverConfig) loadFromEnvs() {
 	if globalIsEnvCompression {
 		s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
 	}
+
+	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
+		if u, err := xnet.ParseURL(jwksURL); err == nil {
+			s.OpenID.JWKS.URL = u
+			s.OpenID.JWKS.PopulatePublicKey()
+		}
+	}
+
+	if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
+		if u, err := xnet.ParseURL(opaURL); err == nil {
+			s.Policy.OPA.URL = u
+			s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN")
+		}
+	}
 }
 
 // TestNotificationTargets tries to establish connections to all notification
@@ -503,17 +520,31 @@ func (s *serverConfig) loadToCachedConfigs() {
 			globalKMSKeyID = globalKMSConfig.Vault.Key.Name
 		}
 	}
+
 	if !globalIsCompressionEnabled {
 		compressionConf := s.GetCompressionConfig()
 		globalCompressExtensions = compressionConf.Extensions
 		globalCompressMimeTypes = compressionConf.MimeTypes
 		globalIsCompressionEnabled = compressionConf.Enabled
 	}
+
+	if globalIAMValidators == nil {
+		globalIAMValidators = getAuthValidators(s)
+	}
+
+	if globalPolicyOPA == nil {
+		if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
+			globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
+				URL:       s.Policy.OPA.URL,
+				AuthToken: s.Policy.OPA.AuthToken,
+			})
+		}
+	}
 }
 
-// newConfig - initialize a new server config, saves env parameters if
+// newSrvConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
-func newConfig(objAPI ObjectLayer) error {
+func newSrvConfig(objAPI ObjectLayer) error {
 	// Initialize server config.
 	srvCfg := newServerConfig()
 
@@ -564,6 +595,20 @@ func loadConfig(objAPI ObjectLayer) error {
 	return nil
 }
 
+// getAuthValidators - returns ValidatorList which contains
+// enabled providers in server config.
+// A new authentication provider is added like below
+// * Add a new provider in pkg/iam/validator package.
+func getAuthValidators(config *serverConfig) *validator.Validators {
+	validators := validator.NewValidators()
+
+	if config.OpenID.JWKS.URL != nil {
+		validators.Add(validator.NewJWT(config.OpenID.JWKS))
+	}
+
+	return validators
+}
+
 // getNotificationTargets - returns TargetList which contains enabled targets in serverConfig.
 // A new notification target is added like below
 // * Add a new target in pkg/event/target package.

cmd/config-current_test.go

@@ -237,7 +237,7 @@ func TestValidateConfig(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		if err = saveConfig(objLayer, configPath, []byte(testCase.configData)); err != nil {
+		if err = saveConfig(context.Background(), objLayer, configPath, []byte(testCase.configData)); err != nil {
 			t.Fatal(err)
 		}
 		_, err = getValidConfig(objLayer)
@@ -260,8 +260,16 @@ func TestConfigDiff(t *testing.T) {
 		{&serverConfig{}, nil, "Given configuration is empty"},
 		// 2
 		{
-			&serverConfig{Credential: auth.Credentials{"u1", "p1"}},
-			&serverConfig{Credential: auth.Credentials{"u1", "p2"}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p1",
+				Expiration: timeSentinel,
+			}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p2",
+				Expiration: timeSentinel,
+			}},
 			"Credential configuration differs",
 		},
 		// 3

cmd/config-migrate.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path"
@@ -29,6 +30,8 @@ import (
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
 )
@@ -2410,7 +2413,7 @@ func migrateV27ToV28() error {
 	return nil
 }
 
-// Migrates '.minio.sys/config.json' to v30.
+// Migrates '.minio.sys/config.json' to v31.
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
 		return err
@@ -2418,73 +2421,167 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	if err := migrateV28ToV29MinioSys(objAPI); err != nil {
 		return err
 	}
-	return migrateV29ToV30MinioSys(objAPI)
+	if err := migrateV29ToV30MinioSys(objAPI); err != nil {
+		return err
+	}
+	return migrateV30ToV31MinioSys(objAPI)
 }
 
-func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
+func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
+	data, err := readConfig(context.Background(), objAPI, configFile)
+	if err != nil {
+		return false, nil, err
+	}
+
+	var versionConfig struct {
+		Version string `json:"version"`
+	}
+
+	vcfg := &versionConfig
+	if err = json.Unmarshal(data, vcfg); err != nil {
+		return false, nil, err
+	}
+	return vcfg.Version == version, data, nil
+}
+
+func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	srvConfig, err := readServerConfig(context.Background(), objAPI)
+	ok, data, err := checkConfigVersion(objAPI, configFile, "27")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
 	}
-	if srvConfig.Version != "29" {
+	if !ok {
 		return nil
 	}
 
-	srvConfig.Version = "30"
-	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
-	srvConfig.Compression.Enabled = false
-	srvConfig.Compression.Extensions = globalCompressExtensions
-	srvConfig.Compression.MimeTypes = globalCompressMimeTypes
-	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from  29  to  30 . %v", err)
+	cfg := &serverConfigV28{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
 	}
 
-	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
+	cfg.Version = "28"
+	cfg.KMS = crypto.KMSConfig{}
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
 	return nil
 }
 
 func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	srvConfig, err := readServerConfig(context.Background(), objAPI)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "28")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
 	}
-	if srvConfig.Version != "28" {
+	if !ok {
 		return nil
 	}
 
-	srvConfig.Version = "29"
-	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
+	cfg := &serverConfigV29{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	cfg.Version = "29"
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "28", "29")
 	return nil
 }
 
-func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
+func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	srvConfig, err := readServerConfig(context.Background(), objAPI)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "29")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
 	}
-	if srvConfig.Version != "27" {
+	if !ok {
 		return nil
 	}
 
-	srvConfig.Version = "28"
-	srvConfig.KMS = crypto.KMSConfig{}
-	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
+	cfg := &serverConfigV30{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
 	}
 
-	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
+	cfg.Version = "30"
+	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
+	cfg.Compression.Enabled = false
+	cfg.Compression.Extensions = globalCompressExtensions
+	cfg.Compression.MimeTypes = globalCompressMimeTypes
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
+	return nil
+}
+
+func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "30")
+	if err == errConfigNotFound {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+	if !ok {
+		return nil
+	}
+
+	cfg := &serverConfigV31{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	cfg.Version = "31"
+	cfg.OpenID.JWKS = validator.JWKSArgs{
+		URL: &xnet.URL{},
+	}
+	cfg.Policy.OPA = iampolicy.OpaArgs{
+		URL:       &xnet.URL{},
+		AuthToken: "",
+	}
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "30", "31")
 	return nil
 }

cmd/config-migrate_test.go

@@ -159,8 +159,8 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	}
 }
 
-// Test if a config migration from v2 to v29 is successfully done
-func TestServerConfigMigrateV2toV29(t *testing.T) {
+// Test if a config migration from v2 to v30 is successfully done
+func TestServerConfigMigrateV2toV30(t *testing.T) {
 	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
 	if err != nil {
 		t.Fatal(err)
@@ -222,6 +222,7 @@ func TestServerConfigMigrateV2toV29(t *testing.T) {
 	if globalServerConfig.Credential.AccessKey != accessKey {
 		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, globalServerConfig.Credential.AccessKey)
 	}
+
 	if globalServerConfig.Credential.SecretKey != secretKey {
 		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, globalServerConfig.Credential.SecretKey)
 	}

cmd/config-versions.go

@@ -22,6 +22,8 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -755,6 +757,9 @@ type serverConfigV28 struct {
 	Logger loggerConfig `json:"logger"`
 }
 
+// serverConfigV29 is just like version '28'.
+type serverConfigV29 serverConfigV28
+
 // compressionConfig represents the compression settings.
 type compressionConfig struct {
 	Enabled    bool     `json:"enabled"`
@@ -765,8 +770,6 @@ type compressionConfig struct {
 // serverConfigV30 is just like version '29', stores additionally
 // extensions and mimetypes fields for compression.
 type serverConfigV30 struct {
-	quick.Config `json:"-"` // ignore interfaces
-
 	Version string `json:"version"`
 
 	// S3 API configuration.
@@ -792,3 +795,45 @@ type serverConfigV30 struct {
 	// Compression configuration
 	Compression compressionConfig `json:"compress"`
 }
+
+// serverConfigV31 is just like version '30', with OPA and OpenID configuration.
+type serverConfigV31 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+
+	// Compression configuration
+	Compression compressionConfig `json:"compress"`
+
+	// OpenID configuration
+	OpenID struct {
+		// JWKS validator config.
+		JWKS validator.JWKSArgs `json:"jwks"`
+	} `json:"openid"`
+
+	// External policy enforcements.
+	Policy struct {
+		// OPA configuration.
+		OPA iampolicy.OpaArgs `json:"opa"`
+
+		// Add new external policy enforcements here.
+	} `json:"policy"`
+}

cmd/config.go

@@ -20,16 +20,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
-	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -62,16 +59,10 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 	}
 
 	// Create a backup of the current config
-	reader, err := readConfig(ctx, objAPI, configFile)
+	oldData, err := readConfig(ctx, objAPI, configFile)
 	if err == nil {
-		var oldData []byte
-		oldData, err = ioutil.ReadAll(reader)
-		if err != nil {
-			return err
-		}
 		backupConfigFile := path.Join(minioConfigPrefix, minioConfigBackupFile)
-		err = saveConfig(objAPI, backupConfigFile, oldData)
-		if err != nil {
+		if err = saveConfig(ctx, objAPI, backupConfigFile, oldData); err != nil {
 			return err
 		}
 	} else {
@@ -81,40 +72,18 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 	}
 
 	// Save the new config in the std config path
-	return saveConfig(objAPI, configFile, data)
-}
-
-func readConfigEtcd(configFile string) ([]byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	resp, err := globalEtcdClient.Get(ctx, configFile)
-	defer cancel()
-	if err != nil {
-		return nil, err
-	}
-	if resp.Count == 0 {
-		return nil, errConfigNotFound
-	}
-	for _, ev := range resp.Kvs {
-		if string(ev.Key) == configFile {
-			return ev.Value, nil
-		}
-	}
-	return nil, errConfigNotFound
+	return saveConfig(ctx, objAPI, configFile, data)
 }
 
 func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
 	var configData []byte
 	var err error
+
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	if globalEtcdClient != nil {
-		configData, err = readConfigEtcd(configFile)
+		configData, err = readConfigEtcd(ctx, globalEtcdClient, configFile)
 	} else {
-		var reader io.Reader
-		reader, err = readConfig(ctx, objAPI, configFile)
-		if err != nil {
-			return nil, err
-		}
-		configData, err = ioutil.ReadAll(reader)
+		configData, err = readConfig(ctx, objAPI, configFile)
 	}
 	if err != nil {
 		return nil, err
@@ -129,82 +98,17 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, e
 	}
 
 	var config = &serverConfig{}
-	if err := json.Unmarshal(configData, config); err != nil {
+	if err = json.Unmarshal(configData, config); err != nil {
 		return nil, err
 	}
 
-	if err := quick.CheckData(config); err != nil {
+	if err = quick.CheckData(config); err != nil {
 		return nil, err
 	}
 
 	return config, nil
 }
 
-func checkServerConfigEtcd(configFile string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	resp, err := globalEtcdClient.Get(ctx, configFile)
-	defer cancel()
-	if err != nil {
-		return err
-	}
-	if resp.Count == 0 {
-		return errConfigNotFound
-	}
-	return nil
-}
-
-func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	if globalEtcdClient != nil {
-		return checkServerConfigEtcd(configFile)
-	}
-
-	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
-		// Convert ObjectNotFound to errConfigNotFound
-		if isErrObjectNotFound(err) {
-			return errConfigNotFound
-		}
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
-		return err
-	}
-	return nil
-}
-
-func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
-	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
-	if err != nil {
-		return err
-	}
-
-	_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil, ObjectOptions{})
-	return err
-}
-
-var errConfigNotFound = errors.New("config file not found")
-
-func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
-	var buffer bytes.Buffer
-	// Read entire content by setting size to -1
-	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
-		// Convert ObjectNotFound and IncompleteBody errors into errConfigNotFound
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, errConfigNotFound
-		}
-
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
-		return nil, err
-	}
-
-	// Return config not found on empty content.
-	if buffer.Len() == 0 {
-		return nil, errConfigNotFound
-	}
-
-	return &buffer, nil
-}
-
 // ConfigSys - config system.
 type ConfigSys struct{}
 
@@ -234,7 +138,8 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 		case _ = <-retryTimerCh:
 			err := initConfig(objAPI)
 			if err != nil {
-				if isInsufficientReadQuorum(err) || isInsufficientWriteQuorum(err) {
+				if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
 					logger.Info("Waiting for configuration to be initialized..")
 					continue
 				}
@@ -255,8 +160,9 @@ func NewConfigSys() *ConfigSys {
 func migrateConfigToMinioSys(objAPI ObjectLayer) error {
 	defer os.Rename(getConfigFile(), getConfigFile()+".deprecated")
 
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	// Verify if backend already has the file.
-	if err := checkServerConfig(context.Background(), objAPI); err != errConfigNotFound {
+	if err := checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
@@ -285,7 +191,14 @@ func initConfig(objAPI ObjectLayer) error {
 		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
 		cancel()
 		if err == nil && resp.Count > 0 {
-			return migrateConfig()
+			if err = migrateConfig(); err != nil {
+				return err
+			}
+
+			// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
+			if err := migrateConfigToMinioSys(objAPI); err != nil {
+				return err
+			}
 		}
 	} else {
 		if isFile(getConfigFile()) {
@@ -301,10 +214,15 @@ func initConfig(objAPI ObjectLayer) error {
 		}
 	}
 
-	if err := checkServerConfig(context.Background(), objAPI); err != nil {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	// Watch config for changes and reloads them in-memory.
+	go watchConfig(objAPI, configFile, loadConfig)
+
+	if err := checkConfig(context.Background(), objAPI, configFile); err != nil {
 		if err == errConfigNotFound {
 			// Config file does not exist, we create it fresh and return upon success.
-			if err = newConfig(objAPI); err != nil {
+			if err = newSrvConfig(objAPI); err != nil {
 				return err
 			}
 		} else {

cmd/crypto/key.go

@@ -140,3 +140,37 @@ func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
 	mac.Sum(partKey[:0])
 	return partKey
 }
+
+// SealETag seals the etag using the object key.
+// It does not encrypt empty ETags because such ETags indicate
+// that the S3 client hasn't sent an ETag = MD5(object) and
+// the backend can pick an ETag value.
+func (key ObjectKey) SealETag(etag []byte) []byte {
+	if len(etag) == 0 { // don't encrypt empty ETag - only if client sent ETag = MD5(object)
+		return etag
+	}
+	var buffer bytes.Buffer
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write([]byte("SSE-etag"))
+	if _, err := sio.Encrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to encrypt ETag using object key"))
+	}
+	return buffer.Bytes()
+}
+
+// UnsealETag unseals the etag using the provided object key.
+// It does not try to decrypt the ETag if len(etag) == 16
+// because such ETags indicate that the S3 client hasn't sent
+// an ETag = MD5(object) and the backend has picked an ETag value.
+func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
+	if !IsETagSealed(etag) {
+		return etag, nil
+	}
+	var buffer bytes.Buffer
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write([]byte("SSE-etag"))
+	if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}

cmd/crypto/key_test.go

@@ -166,3 +166,31 @@ func TestDerivePartKey(t *testing.T) {
 		}
 	}
 }
+
+var sealUnsealETagTests = []string{
+	"",
+	"90682b8e8cc7609c",
+	"90682b8e8cc7609c4671e1d64c73fc30",
+	"90682b8e8cc7609c4671e1d64c73fc307fb3104f",
+}
+
+func TestSealETag(t *testing.T) {
+	var key ObjectKey
+	for i := range key {
+		key[i] = byte(i)
+	}
+	for i, etag := range sealUnsealETagTests {
+		tag, err := hex.DecodeString(etag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decode etag: %s", i, err)
+		}
+		sealedETag := key.SealETag(tag)
+		unsealedETag, err := key.UnsealETag(sealedETag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decrypt etag: %s", i, err)
+		}
+		if !bytes.Equal(unsealedETag, tag) {
+			t.Errorf("Test %d: unsealed etag does not match: got %s - want %s", i, hex.EncodeToString(unsealedETag), etag)
+		}
+	}
+}

cmd/crypto/metadata.go

@@ -40,6 +40,18 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
 	delete(metadata, SSECopyKey)
 }
 
+// RemoveInternalEntries removes all crypto-specific internal
+// metadata entries from the metadata map.
+func RemoveInternalEntries(metadata map[string]string) {
+	delete(metadata, SSEMultipart)
+	delete(metadata, SSEIV)
+	delete(metadata, SSESealAlgorithm)
+	delete(metadata, SSECSealedKey)
+	delete(metadata, S3SealedKey)
+	delete(metadata, S3KMSKeyID)
+	delete(metadata, S3KMSSealedKey)
+}
+
 // IsEncrypted returns true if the object metadata indicates
 // that it was uploaded using some form of server-side-encryption.
 //
@@ -219,3 +231,6 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	copy(sealedKey.Key[:], encryptedKey)
 	return sealedKey, nil
 }
+
+// IsETagSealed returns true if the etag seems to be encrypted.
+func IsETagSealed(etag []byte) bool { return len(etag) > 16 }

cmd/crypto/metadata_test.go

@@ -17,6 +17,7 @@ package crypto
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/hex"
 	"testing"
 
 	"github.com/minio/minio/cmd/logger"
@@ -364,3 +365,75 @@ func TestSSECCreateMetadata(t *testing.T) {
 	}()
 	_ = SSEC.CreateMetadata(nil, SealedKey{Algorithm: InsecureSealAlgorithm})
 }
+
+var isETagSealedTests = []struct {
+	ETag     string
+	IsSealed bool
+}{
+	{ETag: "", IsSealed: false},                                                                                                // 0
+	{ETag: "90682b8e8cc7609c4671e1d64c73fc30", IsSealed: false},                                                                // 1
+	{ETag: "f201040c9dc593e39ea004dc1323699bcd", IsSealed: true},                                                               // 2 not valid ciphertext but looks like sealed ETag
+	{ETag: "20000f00fba2ee2ae4845f725964eeb9e092edfabc7ab9f9239e8344341f769a51ce99b4801b0699b92b16a72fa94972", IsSealed: true}, // 3
+}
+
+func TestIsETagSealed(t *testing.T) {
+	for i, test := range isETagSealedTests {
+		etag, err := hex.DecodeString(test.ETag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decode etag: %s", i, err)
+		}
+		if sealed := IsETagSealed(etag); sealed != test.IsSealed {
+			t.Errorf("Test %d: got %v - want %v", i, sealed, test.IsSealed)
+		}
+	}
+}
+
+var removeInternalEntriesTests = []struct {
+	Metadata, Expected map[string]string
+}{
+	{ // 0
+		Metadata: map[string]string{
+			SSEMultipart:     "",
+			SSEIV:            "",
+			SSESealAlgorithm: "",
+			SSECSealedKey:    "",
+			S3SealedKey:      "",
+			S3KMSKeyID:       "",
+			S3KMSSealedKey:   "",
+		},
+		Expected: map[string]string{},
+	},
+	{ // 1
+		Metadata: map[string]string{
+			SSEMultipart:         "",
+			SSEIV:                "",
+			"X-Amz-Meta-A":       "X",
+			"X-Minio-Internal-B": "Y",
+		},
+		Expected: map[string]string{
+			"X-Amz-Meta-A":       "X",
+			"X-Minio-Internal-B": "Y",
+		},
+	},
+}
+
+func TestRemoveInternalEntries(t *testing.T) {
+	isEqual := func(x, y map[string]string) bool {
+		if len(x) != len(y) {
+			return false
+		}
+		for k, v := range x {
+			if u, ok := y[k]; !ok || v != u {
+				return false
+			}
+		}
+		return true
+	}
+
+	for i, test := range removeInternalEntriesTests {
+		RemoveInternalEntries(test.Metadata)
+		if !isEqual(test.Metadata, test.Expected) {
+			t.Errorf("Test %d: got %v - want %v", i, test.Metadata, test.Expected)
+		}
+	}
+}

cmd/crypto/sse.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"errors"
 	"io"
+	"net/http"
+	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
@@ -69,10 +71,59 @@ const (
 // domain is "SSE-S3".
 func (s3) String() string { return "SSE-S3" }
 
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (sse s3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := sse.ParseMetadata(metadata)
+	if err != nil {
+		return
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return
+	}
+	err = key.Unseal(unsealKey, sealedKey, sse.String(), bucket, object)
+	return
+}
+
 // String returns the SSE domain as string. For SSE-C the
 // domain is "SSE-C".
 func (ssec) String() string { return "SSE-C" }
 
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-C client key of the HTTP headers
+// and returns the decrypted object key.
+func (sse ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := sse.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-Copy client key of the HTTP headers
+// and returns the decrypted object key.
+func (sse ssecCopy) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := sse.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// unsealObjectKey decrypts and returns the sealed object key
+// from the metadata using the SSE-C client key.
+func unsealObjectKey(clientKey [32]byte, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	sealedKey, err := SSEC.ParseMetadata(metadata)
+	if err != nil {
+		return
+	}
+	err = key.Unseal(clientKey, sealedKey, SSEC.String(), bucket, object)
+	return
+}
+
 // EncryptSinglePart encrypts an io.Reader which must be the
 // the body of a single-part PUT request.
 func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {

cmd/crypto/sse_test.go

@@ -14,7 +14,10 @@
 
 package crypto
 
-import "testing"
+import (
+	"net/http"
+	"testing"
+)
 
 func TestS3String(t *testing.T) {
 	const Domain = "SSE-S3"
@@ -29,3 +32,195 @@ func TestSSECString(t *testing.T) {
 		t.Errorf("SSEC's string method returns wrong domain: got '%s' - want '%s'", domain, Domain)
 	}
 }
+
+var ssecUnsealObjectKeyTests = []struct {
+	Headers        http.Header
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object2",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrSecretKeyMismatch,
+	},
+	{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":         "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+	{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5,
+	},
+}
+
+func TestSSECUnsealObjectKey(t *testing.T) {
+	for i, test := range ssecUnsealObjectKeyTests {
+		if _, err := SSEC.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}
+
+var sseCopyUnsealObjectKeyTests = []struct {
+	Headers        http.Header
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object2",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrSecretKeyMismatch,
+	},
+	{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":         "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+	{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5,
+	},
+}
+
+func TestSSECopyUnsealObjectKey(t *testing.T) {
+	for i, test := range sseCopyUnsealObjectKeyTests {
+		if _, err := SSECopy.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}
+
+var s3UnsealObjectKeyTests = []struct {
+	KMS            KMS
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid KMS key-ID and valid metadata entries for bucket/object
+		KMS:    NewKMS([32]byte{}),
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Iv":                "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":     "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":     "test-key-1",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":    "DAREv2-HMAC-SHA256",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid KMS key-ID for invalid metadata entries for bucket/object
+		KMS:    NewKMS([32]byte{}),
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Iv":                "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":     "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":     "test-key-1",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+}
+
+func TestS3UnsealObjectKey(t *testing.T) {
+	for i, test := range s3UnsealObjectKeyTests {
+		if _, err := S3.UnsealObjectKey(test.KMS, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}

cmd/disk-cache.go

@@ -38,8 +38,7 @@ import (
 )
 
 const (
-	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
-	cacheSizeMultiplier  = 100
+	// disk cache needs to have  object size space free for a cache entry to be created.
 	cacheTrashDir        = "trash"
 	cacheCleanupInterval = 10 // in minutes
 )
@@ -184,10 +183,7 @@ func (c cacheObjects) getMetadata(objInfo ObjectInfo) map[string]string {
 }
 
 func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-
-	objInfo, bkErr := c.GetObjectInfoFn(ctx, bucket, object, opts)
-
-	if c.isCacheExclude(bucket, object) || !objInfo.IsCacheable() {
+	if c.isCacheExclude(bucket, object) {
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
 
@@ -197,30 +193,29 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
 
-	backendDown := backendDownError(bkErr)
-	if bkErr != nil && !backendDown {
+	cacheReader, cacheErr := dcache.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+
+	objInfo, err := c.GetObjectInfoFn(ctx, bucket, object, opts)
+	if backendDownError(err) && cacheErr == nil {
+		return cacheReader, nil
+	} else if err != nil {
 		if _, ok := err.(ObjectNotFound); ok {
-			// Delete the cached entry if backend object was deleted.
+			// Delete cached entry if backend object was deleted.
 			dcache.Delete(ctx, bucket, object)
 		}
-		return nil, bkErr
+		return nil, err
 	}
 
-	if !backendDown && filterFromCache(objInfo.UserDefined) {
+	if !objInfo.IsCacheable() || filterFromCache(objInfo.UserDefined) {
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
 
-	if cacheReader, cacheErr := dcache.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts); cacheErr == nil {
-		if backendDown {
-			// If the backend is down, serve the request from cache.
-			return cacheReader, nil
-		}
-
+	if cacheErr == nil {
 		if cacheReader.ObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) {
 			// Object is not stale, so serve from cache
 			return cacheReader, nil
 		}
-
+		cacheReader.Close()
 		// Object is stale, so delete from cache
 		dcache.Delete(ctx, bucket, object)
 	}
@@ -232,8 +227,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		// We don't cache partial objects.
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
-	if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
-		// cache only objects < 1/100th of disk capacity
+	if !dcache.diskAvailable(objInfo.Size) {
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
 
@@ -290,6 +284,10 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		return err
 	}
 
+	if !backendDown && !objInfo.IsCacheable() {
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
+	}
+
 	if !backendDown && filterFromCache(objInfo.UserDefined) {
 		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
@@ -309,8 +307,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		// We don't cache partial objects.
 		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
-	if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
-		// cache only objects < 1/100th of disk capacity
+	if !dcache.diskAvailable(objInfo.Size) {
 		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// Initialize pipe.
@@ -657,7 +654,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 	size := r.Size()
 
 	// fetch from backend if there is no space on cache drive
-	if !dcache.diskAvailable(size * cacheSizeMultiplier) {
+	if !dcache.diskAvailable(size) {
 		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	// fetch from backend if cache exclude pattern or cache-control
@@ -749,9 +746,9 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
-	// make sure cache has at least cacheSizeMultiplier * size available
+	// make sure cache has at least size space available
 	size := data.Size()
-	if !dcache.diskAvailable(size * cacheSizeMultiplier) {
+	if !dcache.diskAvailable(size) {
 		select {
 		case dcache.purgeChan <- struct{}{}:
 		default:
@@ -969,6 +966,9 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
+		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+		},
 		PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts)
 		},

cmd/encryption-v1.go

@@ -37,7 +37,6 @@ import (
 
 var (
 	// AWS errors for invalid SSE-C requests.
-	errInsecureSSERequest   = errors.New("SSE-C requests require TLS connections")
 	errEncryptedObject      = errors.New("The object was stored using a form of SSE")
 	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
 	errKMSNotConfigured     = errors.New("KMS not configured for a server side encrypted object")
@@ -80,16 +79,31 @@ func hasServerSideEncryptionHeader(header http.Header) bool {
 	return crypto.S3.IsRequested(header) || crypto.SSEC.IsRequested(header)
 }
 
+// isEncryptedMultipart returns true if the current object is
+// uploaded by the user using multipart mechanism:
+// initiate new multipart, upload part, complete upload
+func isEncryptedMultipart(objInfo ObjectInfo) bool {
+	if len(objInfo.Parts) == 0 {
+		return false
+	}
+	if !crypto.IsMultiPart(objInfo.UserDefined) {
+		return false
+	}
+	for _, part := range objInfo.Parts {
+		_, err := sio.DecryptedSize(uint64(part.Size))
+		if err != nil {
+			return false
+		}
+	}
+	// Further check if this object is uploaded using multipart mechanism
+	// by the user and it is not about XL internally splitting the
+	// object into parts in PutObject()
+	return !(objInfo.backendType == BackendErasure && len(objInfo.ETag) == 32)
+}
+
 // ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.
 // It returns the client provided key on success.
 func ParseSSECopyCustomerRequest(h http.Header, metadata map[string]string) (key []byte, err error) {
-	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
-		// we cannot use r.TLS == nil here because Go's http implementation reflects on
-		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
-		// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
-		// will always fail -> r.TLS is always nil even for TLS requests.
-		return nil, errInsecureSSERequest
-	}
 	if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(h) {
 		return nil, crypto.ErrIncompatibleEncryptionMethod
 	}
@@ -106,13 +120,6 @@ func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {
 // ParseSSECustomerHeader parses the SSE-C header fields and returns
 // the client provided key on success.
 func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
-	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
-		// we cannot use r.TLS == nil here because Go's http implementation reflects on
-		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
-		// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
-		// will always fail -> r.TLS is always nil even for TLS requests.
-		return nil, errInsecureSSERequest
-	}
 	if crypto.S3.IsRequested(header) && crypto.SSEC.IsRequested(header) {
 		return key, crypto.ErrIncompatibleEncryptionMethod
 	}
@@ -149,6 +156,30 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 		sealedKey = objectKey.Seal(extKey, sealedKey.IV, crypto.SSEC.String(), bucket, object)
 		crypto.SSEC.CreateMetadata(metadata, sealedKey)
 		return nil
+	case crypto.S3.IsEncrypted(metadata):
+		if globalKMS == nil {
+			return errKMSNotConfigured
+		}
+		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
+		if err != nil {
+			return err
+		}
+		oldKey, err := globalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
+		if err != nil {
+			return err
+		}
+		var objectKey crypto.ObjectKey
+		if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
+			return err
+		}
+
+		newKey, encKey, err := globalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+		if err != nil {
+			return err
+		}
+		sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
+		crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
+		return nil
 	}
 }
 
@@ -325,7 +356,7 @@ func DecryptRequestWithSequenceNumberR(client io.Reader, h http.Header, bucket,
 
 // DecryptCopyRequestR - same as DecryptCopyRequest, but with a
 // Reader
-func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, metadata map[string]string) (io.Reader, error) {
+func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
 	var (
 		key []byte
 		err error
@@ -336,7 +367,7 @@ func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string,
 			return nil, err
 		}
 	}
-	return newDecryptReader(client, key, bucket, object, 0, metadata)
+	return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
 }
 
 func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
@@ -358,17 +389,6 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
 	return reader, nil
 }
 
-// GetEncryptedOffsetLength - returns encrypted offset and length
-// along with sequence number
-func GetEncryptedOffsetLength(startOffset, length int64, objInfo ObjectInfo) (seqNumber uint32, encStartOffset, encLength int64) {
-	if len(objInfo.Parts) == 0 || !crypto.IsMultiPart(objInfo.UserDefined) {
-		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
-		return
-	}
-	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
-	return
-}
-
 // DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
 // reader
 func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
@@ -378,11 +398,11 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
 	bucket, object := oi.Bucket, oi.Name
 
 	// Single part case
-	if len(oi.Parts) == 0 || !crypto.IsMultiPart(oi.UserDefined) {
+	if !isEncryptedMultipart(oi) {
 		var reader io.Reader
 		var err error
 		if copySource {
-			reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, oi.UserDefined)
+			reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
 		} else {
 			reader, err = DecryptRequestWithSequenceNumberR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
 		}
@@ -708,7 +728,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 	var seqNumber uint32
 	var encStartOffset, encLength int64
 
-	if len(objInfo.Parts) == 0 || !crypto.IsMultiPart(objInfo.UserDefined) {
+	if !isEncryptedMultipart(objInfo) {
 		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
 
 		var writer io.WriteCloser
@@ -870,7 +890,7 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if !crypto.IsEncrypted(o.UserDefined) {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
-	if len(o.Parts) == 0 || !crypto.IsMultiPart(o.UserDefined) {
+	if !isEncryptedMultipart(*o) {
 		size, err := sio.DecryptedSize(uint64(o.Size))
 		if err != nil {
 			err = errObjectTampered // assign correct error type
@@ -918,7 +938,7 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
 	}
 	sizes := []int64{int64(partSize)}
 	decObjSize = sizes[0]
-	if crypto.IsMultiPart(o.UserDefined) {
+	if isEncryptedMultipart(*o) {
 		sizes = make([]int64, len(o.Parts))
 		decObjSize = 0
 		for i, part := range o.Parts {

cmd/encryption-v1_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"encoding/base64"
 	"net/http"
 	"testing"
 
@@ -107,228 +106,6 @@ func TesthasSSECustomerHeader(t *testing.T) {
 	}
 }
 
-var parseSSECustomerRequestTests = []struct {
-	headers map[string]string
-	useTLS  bool
-	err     error
-}{
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: true, err: nil,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: false, err: errInsecureSSERequest,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES 256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: true, err: crypto.ErrInvalidCustomerAlgorithm,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
-			crypto.SSECKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
-			crypto.SSECKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
-			crypto.SSECKeyMD5:    " +jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
-			crypto.SSECKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		useTLS: true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "", // 7
-			crypto.SSECKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		useTLS: true, err: crypto.ErrMissingCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			crypto.SSECKeyMD5:    "",
-		},
-		useTLS: true, err: crypto.ErrMissingCustomerKeyMD5,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			crypto.SSECKeyMD5:    "",
-			crypto.SSEHeader:     "",
-		},
-		useTLS: true, err: crypto.ErrIncompatibleEncryptionMethod,
-	},
-}
-
-func TestParseSSECustomerRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	for i, test := range parseSSECustomerRequestTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		request := &http.Request{}
-		request.Header = headers
-		globalIsSSL = test.useTLS
-
-		_, err := ParseSSECustomerRequest(request)
-		if err != test.err {
-			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
-		}
-	}
-}
-
-var parseSSECopyCustomerRequestTests = []struct {
-	headers  map[string]string
-	metadata map[string]string
-	useTLS   bool
-	err      error
-}{
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: nil,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": base64.StdEncoding.EncodeToString(make([]byte, 64))},
-		useTLS:   true, err: crypto.ErrIncompatibleEncryptionMethod,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-		useTLS:   false, err: errInsecureSSERequest,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES 256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrInvalidCustomerAlgorithm,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
-			crypto.SSECopyKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
-			crypto.SSECopyKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
-			crypto.SSECopyKeyMD5:    " +jq/LwEOEO90YtiTuNFVw==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
-			crypto.SSECopyKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "", // 7
-			crypto.SSECopyKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrMissingCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			crypto.SSECopyKeyMD5:    "",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrMissingCustomerKeyMD5,
-	},
-}
-
-func TestParseSSECopyCustomerRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	for i, test := range parseSSECopyCustomerRequestTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		request := &http.Request{}
-		request.Header = headers
-		globalIsSSL = test.useTLS
-
-		_, err := ParseSSECopyCustomerRequest(request.Header, test.metadata)
-		if err != test.err {
-			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
-		}
-	}
-}
-
 var encryptRequestTests = []struct {
 	header   map[string]string
 	metadata map[string]string

cmd/erasure-utils.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"strings"
 
 	"github.com/klauspost/reedsolomon"
 	"github.com/minio/minio/cmd/logger"
@@ -82,7 +81,9 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		if write < int64(len(block)) {
 			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
 			if err != nil {
-				logger.LogIf(ctx, err)
+				if err != io.ErrClosedPipe {
+					logger.LogIf(ctx, err)
+				}
 				return 0, err
 			}
 			totalWritten += n
@@ -92,7 +93,7 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		n, err := io.Copy(dst, bytes.NewReader(block))
 		if err != nil {
 			// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
-			if !strings.Contains(err.Error(), "read/write on closed pipe") {
+			if err != io.ErrClosedPipe {
 				logger.LogIf(ctx, err)
 			}
 			return 0, err

cmd/format-xl.go

@@ -379,6 +379,22 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
 	return disk.RenameFile(minioMetaBucket, formatConfigFileTmp, minioMetaBucket, formatConfigFile)
 }
 
+var ignoredHiddenDirectories = []string{
+	minioMetaBucket,
+	"lost+found",
+	"$RECYCLE.BIN",
+	"System Volume Information",
+}
+
+func isIgnoreHiddenDirectories(dir string) bool {
+	for _, ignDir := range ignoredHiddenDirectories {
+		if dir == ignDir {
+			return true
+		}
+	}
+	return false
+}
+
 // loadFormatXL - loads format.json from disk.
 func loadFormatXL(disk StorageAPI) (format *formatXLV3, err error) {
 	buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
@@ -391,9 +407,7 @@ func loadFormatXL(disk StorageAPI) (format *formatXLV3, err error) {
 			if err != nil {
 				return nil, err
 			}
-			if len(vols) > 1 || (len(vols) == 1 &&
-				vols[0].Name != minioMetaBucket &&
-				vols[0].Name != "lost+found") {
+			if len(vols) > 1 || (len(vols) == 1 && !isIgnoreHiddenDirectories(vols[0].Name)) {
 				// 'format.json' not found, but we
 				// found user data.
 				return nil, errCorruptedFormat

cmd/fs-v1-multipart.go

@@ -713,6 +713,11 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 	}
 
 	fs.appendFileMapMu.Lock()
+	// Remove file in tmp folder
+	file := fs.appendFileMap[uploadID]
+	if file != nil {
+		fsRemoveFile(ctx, file.filePath)
+	}
 	delete(fs.appendFileMap, uploadID)
 	fs.appendFileMapMu.Unlock()
 
@@ -725,10 +730,14 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 		}
 		return toObjectErr(err, bucket, object)
 	}
+
 	// Ignore the error returned as Windows fails to remove directory if a file in it
 	// is Open()ed by the backgroundAppend()
 	fsRemoveAll(ctx, uploadIDDir)
 
+	// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
+	fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
+
 	return nil
 }
 

cmd/fs-v1.go

@@ -1018,7 +1018,7 @@ func (fs *FSObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
 	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
-		if err != nil {
+		if err != nil && err != errFileNotFound {
 			logger.LogIf(context.Background(), err)
 			return
 		}
@@ -1274,7 +1274,7 @@ func (fs *FSObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
 
 // SetBucketPolicy sets policy on bucket
 func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
-	return savePolicyConfig(fs, bucket, policy)
+	return savePolicyConfig(ctx, fs, bucket, policy)
 }
 
 // GetBucketPolicy will get policy on bucket

cmd/gateway-common.go

@@ -224,7 +224,16 @@ func FromMinioClientListBucketResultToV2Info(bucket string, result minio.ListBuc
 	}
 }
 
-// ToMinioClientMetadata converts metadata to map[string][]string
+// ToMinioClientObjectInfoMetadata convertes metadata to map[string][]string
+func ToMinioClientObjectInfoMetadata(metadata map[string]string) map[string][]string {
+	mm := make(map[string][]string, len(metadata))
+	for k, v := range metadata {
+		mm[http.CanonicalHeaderKey(k)] = []string{v}
+	}
+	return mm
+}
+
+// ToMinioClientMetadata converts metadata to map[string]string
 func ToMinioClientMetadata(metadata map[string]string) map[string]string {
 	mm := make(map[string]string)
 	for k, v := range metadata {
@@ -285,6 +294,8 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		err = BucketPolicyNotFound{}
 	case "InvalidBucketName":
 		err = BucketNameInvalid{Bucket: bucket}
+	case "InvalidPart":
+		err = InvalidPart{}
 	case "NoSuchBucket":
 		err = BucketNotFound{Bucket: bucket}
 	case "NoSuchKey":

cmd/gateway-main.go

@@ -134,9 +134,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Handle common command args.
 	handleCommonCmdArgs(ctx)
 
-	// Handle common env vars.
-	handleCommonEnvVars()
-
 	// Get port to listen on from gateway address
 	_, gatewayPort, pErr := net.SplitHostPort(gatewayAddr)
 	if pErr != nil {
@@ -149,54 +146,41 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// To avoid this error situation we check for port availability.
 	logger.FatalIf(checkPortAvailability(gatewayPort), "Unable to start the gateway")
 
+	// Create certs path.
+	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
+
+	// Check and load TLS certificates.
+	var err error
+	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
+	logger.FatalIf(err, "Invalid TLS certificate file")
+
+	// Check and load Root CAs.
+	globalRootCAs, err = getRootCAs(getCADir())
+	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
+
+	// Handle common env vars.
+	handleCommonEnvVars()
+
 	// Validate if we have access, secret set through environment.
 	if !globalIsEnvCreds {
 		logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
 	}
 
-	// Create certs path.
-	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
-
-	// Initialize server config.
-	srvCfg := newServerConfig()
-
-	// Override any values from ENVs.
-	srvCfg.loadFromEnvs()
-
-	// Load values to cached global values.
-	srvCfg.loadToCachedConfigs()
-
-	// hold the mutex lock before a new config is assigned.
-	globalServerConfigMu.Lock()
-	globalServerConfig = srvCfg
-	globalServerConfigMu.Unlock()
-
-	var cacheConfig = globalServerConfig.GetCacheConfig()
-	if len(cacheConfig.Drives) > 0 {
-		var err error
-		// initialize the new disk cache objects.
-		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
-		logger.FatalIf(err, "Unable to initialize disk caching")
-	}
-
-	// Check and load SSL certificates.
-	var err error
-	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
-	logger.FatalIf(err, "Invalid SSL certificate file")
-
 	// Set system resources to maximum.
 	logger.LogIf(context.Background(), setMaxResources())
 
 	initNSLock(false) // Enable local namespace lock.
 
-	// Create new notification system.
-	globalNotificationSys = NewNotificationSys(globalServerConfig, EndpointList{})
-
-	// Create new policy system.
-	globalPolicySys = NewPolicySys()
-
 	router := mux.NewRouter().SkipClean(true)
 
+	if globalEtcdClient != nil {
+		// Enable STS router if etcd is enabled.
+		registerSTSRouter(router)
+
+		// Enable admin router if etcd is enabled.
+		registerAdminRouter(router)
+	}
+
 	// Add healthcheck router
 	registerHealthCheckRouter(router)
 
@@ -234,8 +218,56 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 
+	// Create a new config system.
+	globalConfigSys = NewConfigSys()
+
+	// Initialize server config.
+	srvCfg := newServerConfig()
+
+	// Override any values from ENVs.
+	srvCfg.loadFromEnvs()
+
+	// Load values to cached global values.
+	srvCfg.loadToCachedConfigs()
+
+	// hold the mutex lock before a new config is assigned.
+	globalServerConfigMu.Lock()
+	globalServerConfig = srvCfg
+	globalServerConfigMu.Unlock()
+
+	// Load logger subsystem
+	loadLoggers()
+
+	// This is only to uniquely identify each gateway deployments.
+	logger.SetDeploymentID(os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID"))
+
+	var cacheConfig = globalServerConfig.GetCacheConfig()
+	if len(cacheConfig.Drives) > 0 {
+		var err error
+		// initialize the new disk cache objects.
+		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
+		logger.FatalIf(err, "Unable to initialize disk caching")
+	}
+
+	// Re-enable logging
+	logger.Disable = false
+
+	// Create new IAM system.
+	globalIAMSys = NewIAMSys()
+	if globalEtcdClient != nil {
+		// Initialize IAM sys.
+		go globalIAMSys.Init(newObject)
+	}
+
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
+
+	// Initialize policy system.
 	go globalPolicySys.Init(newObject)
 
+	// Create new notification system.
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+
 	// Once endpoints are finalized, initialize the new object api.
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject
@@ -256,8 +288,5 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
 	}
 
-	// Reenable logging
-	logger.Disable = false
-
 	handleSignals()
 }

cmd/gateway-startup-msg.go

@@ -57,8 +57,10 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 	// Colorize the message and print.
 	logger.StartupMessage(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
-	logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
-	logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+	if isTerminal() {
+		logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+	}
 
 	if globalIsBrowserEnabled {
 		logger.StartupMessage(colorBlue("\nBrowser Access:"))

cmd/gateway/s3/gateway-s3.go

@@ -71,6 +71,9 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
      MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
+  LOGGER:
+     MINIO_LOGGER_HTTP_ENDPOINT: HTTP endpoint URL to log all incoming requests.
+
 EXAMPLES:
   1. Start minio gateway server for AWS S3 backend.
      $ export MINIO_ACCESS_KEY=accesskey
@@ -82,7 +85,13 @@ EXAMPLES:
      $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
      $ {{.HelpName}} https://play.minio.io:9000
 
-  3. Start minio gateway server for AWS S3 backend with edge caching enabled.
+  3. Start minio gateway server for AWS S3 backend logging all requests to http endpoint.
+     $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
+     $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+     $ export MINIO_LOGGER_HTTP_ENDPOINT="http://localhost:8000/"
+     $ {{.HelpName}} https://play.minio.io:9000
+
+  4. Start minio gateway server for AWS S3 backend with edge caching enabled.
      $ export MINIO_ACCESS_KEY=accesskey
      $ export MINIO_SECRET_KEY=secretkey
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
@@ -194,6 +203,9 @@ func newS3(url string) (*miniogo.Core, error) {
 		return nil, err
 	}
 
+	// Set custom transport
+	clnt.SetCustomTransport(minio.NewCustomHTTPTransport())
+
 	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
 	// Check if the provided keys are valid.
 	if _, err = clnt.BucketExists(probeBucketName); err != nil {
@@ -332,13 +344,13 @@ func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 	var objInfo minio.ObjectInfo
 	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
-		return nil, err
+		return nil, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
 	var startOffset, length int64
 	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
 	if err != nil {
-		return nil, err
+		return nil, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
 	pr, pw := io.Pipe()
@@ -368,7 +380,6 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 
 	if startOffset >= 0 && length >= 0 {
 		if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
-			logger.LogIf(ctx, err)
 			return minio.ErrorRespToObjectError(err, bucket, key)
 		}
 	}
@@ -379,7 +390,6 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 	defer object.Close()
 
 	if _, err := io.Copy(writer, object); err != nil {
-		logger.LogIf(ctx, err)
 		return minio.ErrorRespToObjectError(err, bucket, key)
 	}
 	return nil
@@ -401,6 +411,9 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string,
 	if err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
 	}
+	// On success, populate the key & metadata so they are present in the notification
+	oi.Key = object
+	oi.Metadata = minio.ToMinioClientObjectInfoMetadata(metadata)
 
 	return minio.FromMinioClientObjectInfo(bucket, oi), nil
 }
@@ -482,7 +495,7 @@ func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, de
 func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, e error) {
 	result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
-		return lpi, err
+		return lpi, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
 	return minio.FromMinioClientListPartsInfo(result), nil

cmd/gateway/s3/gateway-s3_test.go

@@ -50,6 +50,10 @@ func TestS3ToObjectError(t *testing.T) {
 			inputErr:    errResponse("InvalidBucketName"),
 			expectedErr: minio.BucketNameInvalid{},
 		},
+		{
+			inputErr:    errResponse("InvalidPart"),
+			expectedErr: minio.InvalidPart{},
+		},
 		{
 			inputErr:    errResponse("NoSuchBucketPolicy"),
 			expectedErr: minio.BucketPolicyNotFound{},

cmd/generic-handlers.go

@@ -28,6 +28,7 @@ import (
 	"github.com/minio/minio-go/pkg/set"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/handlers"
@@ -773,3 +774,21 @@ func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	}()
 	h.handler.ServeHTTP(w, r)
 }
+
+func setSSETLSHandler(h http.Handler) http.Handler { return sseTLSHandler{h} }
+
+// sseTLSHandler enforces certain rules for SSE requests which are made / must be made over TLS.
+type sseTLSHandler struct{ handler http.Handler }
+
+func (h sseTLSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Deny SSE-C requests if not made over TLS
+	if !globalIsSSL && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
+		if r.Method == http.MethodHead {
+			writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
+		} else {
+			writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL)
+		}
+		return
+	}
+	h.handler.ServeHTTP(w, r)
+}

cmd/generic-handlers_test.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	"net/http"
+	"net/http/httptest"
+	"net/url"
 	"strconv"
 	"testing"
 
@@ -181,3 +183,41 @@ func TestContainsReservedMetadata(t *testing.T) {
 		}
 	}
 }
+
+var sseTLSHandlerTests = []struct {
+	URL               *url.URL
+	Header            http.Header
+	IsTLS, ShouldFail bool
+}{
+	{URL: &url.URL{}, Header: http.Header{}, IsTLS: false, ShouldFail: false},                                        // 0
+	{URL: &url.URL{}, Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: false, ShouldFail: true}, // 1
+	{URL: &url.URL{}, Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: true, ShouldFail: false}, // 2
+	{URL: &url.URL{}, Header: http.Header{crypto.SSECKey: []string{""}}, IsTLS: true, ShouldFail: false},             // 3
+	{URL: &url.URL{}, Header: http.Header{crypto.SSECopyAlgorithm: []string{""}}, IsTLS: false, ShouldFail: true},    // 4
+}
+
+func TestSSETLSHandler(t *testing.T) {
+	defer func(isSSL bool) { globalIsSSL = isSSL }(globalIsSSL) // reset globalIsSSL after test
+
+	var okHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}
+	for i, test := range sseTLSHandlerTests {
+		globalIsSSL = test.IsTLS
+
+		w := httptest.NewRecorder()
+		r := new(http.Request)
+		r.Header = test.Header
+		r.URL = test.URL
+
+		h := setSSETLSHandler(okHandler)
+		h.ServeHTTP(w, r)
+
+		switch {
+		case test.ShouldFail && w.Code == http.StatusOK:
+			t.Errorf("Test %d: should fail but status code is HTTP %d", i, w.Code)
+		case !test.ShouldFail && w.Code != http.StatusOK:
+			t.Errorf("Test %d: should not fail but status code is HTTP %d and not 200 OK", i, w.Code)
+		}
+	}
+}

cmd/globals.go

@@ -34,6 +34,8 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/dns"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
 )
 
 // minio configuration related constants.
@@ -81,6 +83,8 @@ const (
 	globalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
 	// Refresh interval to update in-memory bucket policy cache.
 	globalRefreshBucketPolicyInterval = 5 * time.Minute
+	// Refresh interval to update in-memory iam config cache.
+	globalRefreshIAMInterval = 5 * time.Minute
 
 	// Limit of location constraint XML for unauthenticted PUT bucket operations.
 	maxLocationConstraintSize = 3 * humanize.MiByte
@@ -132,6 +136,7 @@ var (
 
 	globalNotificationSys *NotificationSys
 	globalPolicySys       *PolicySys
+	globalIAMSys          *IAMSys
 
 	// CA root certificates, a nil value means system certs pool will be used
 	globalRootCAs *x509.CertPool
@@ -244,6 +249,12 @@ var (
 	// Some standard content-types which we strictly dis-allow for compression.
 	standardExcludeCompressContentTypes = []string{"video/*", "audio/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", " application/x-compress", "application/x-spoon"}
 
+	// Authorization validators list.
+	globalIAMValidators *validator.Validators
+
+	// OPA policy system.
+	globalPolicyOPA *iampolicy.Opa
+
 	// Add new variable global values here.
 )
 

cmd/handler-utils.go

@@ -176,14 +176,26 @@ func getRedirectPostRawQuery(objInfo ObjectInfo) string {
 	return redirectValues.Encode()
 }
 
+// Returns access key in the request Authorization header.
+func getReqAccessKey(r *http.Request, region string) (accessKey string) {
+	accessKey, _, _ = getReqAccessKeyV4(r, region)
+	if accessKey == "" {
+		accessKey, _, _ = getReqAccessKeyV2(r)
+	}
+	return accessKey
+}
+
 // Extract request params to be sent with event notifiation.
 func extractReqParams(r *http.Request) map[string]string {
 	if r == nil {
 		return nil
 	}
 
+	region := globalServerConfig.GetRegion()
 	// Success.
 	return map[string]string{
+		"region":          region,
+		"accessKey":       getReqAccessKey(r, region),
 		"sourceIPAddress": handlers.GetSourceIP(r),
 		// Add more fields here.
 	}
@@ -193,6 +205,7 @@ func extractReqParams(r *http.Request) map[string]string {
 func extractRespElements(w http.ResponseWriter) map[string]string {
 
 	return map[string]string{
+		"requestId":      w.Header().Get(responseRequestIDKey),
 		"content-length": w.Header().Get("Content-Length"),
 		// Add more fields here.
 	}

cmd/handler-utils_test.go

@@ -172,6 +172,16 @@ func TestExtractMetadataHeaders(t *testing.T) {
 			},
 			shouldFail: false,
 		},
+		// Support multiple values
+		{
+			header: http.Header{
+				"x-amz-meta-key": []string{"amz-meta1", "amz-meta2"},
+			},
+			metadata: map[string]string{
+				"x-amz-meta-key": "amz-meta1,amz-meta2",
+			},
+			shouldFail: false,
+		},
 		// Empty header input returns empty metadata.
 		{
 			header:     nil,

cmd/http/bufconn_test.go

@@ -46,7 +46,8 @@ func TestBuffConnReadTimeout(t *testing.T) {
 
 		tcpConn, terr := tcpListener.AcceptTCP()
 		if terr != nil {
-			t.Fatalf("failed to accept new connection. %v", terr)
+			t.Errorf("failed to accept new connection. %v", terr)
+			return
 		}
 		bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second)
 		defer bufconn.Close()
@@ -55,11 +56,13 @@ func TestBuffConnReadTimeout(t *testing.T) {
 		var b = make([]byte, 12)
 		_, terr = bufconn.Read(b)
 		if terr != nil {
-			t.Fatalf("failed to read from client. %v", terr)
+			t.Errorf("failed to read from client. %v", terr)
+			return
 		}
 		received := string(b)
 		if received != "message one\n" {
-			t.Fatalf(`server: expected: "message one\n", got: %v`, received)
+			t.Errorf(`server: expected: "message one\n", got: %v`, received)
+			return
 		}
 
 		// Wait for more than read timeout to simulate processing.
@@ -67,17 +70,20 @@ func TestBuffConnReadTimeout(t *testing.T) {
 
 		_, terr = bufconn.Read(b)
 		if terr != nil {
-			t.Fatalf("failed to read from client. %v", terr)
+			t.Errorf("failed to read from client. %v", terr)
+			return
 		}
 		received = string(b)
 		if received != "message two\n" {
-			t.Fatalf(`server: expected: "message two\n", got: %v`, received)
+			t.Errorf(`server: expected: "message two\n", got: %v`, received)
+			return
 		}
 
 		// Send a response.
 		_, terr = io.WriteString(bufconn, "messages received\n")
 		if terr != nil {
-			t.Fatalf("failed to write to client. %v", terr)
+			t.Errorf("failed to write to client. %v", terr)
+			return
 		}
 
 		// Removes all deadlines if any.

cmd/http/listener.go

@@ -141,8 +141,8 @@ func getMethodResourceHost(bufConn *BufConn, maxHeaderBytes int) (method string,
 			// HTTP headers are case insensitive, so we should simply convert
 			// each tokens to their lower case form to match 'host' header.
 			token = strings.ToLower(token)
-			if strings.HasPrefix(token, "host: ") {
-				host = strings.TrimPrefix(strings.TrimSuffix(token, "\r"), "host: ")
+			if strings.HasPrefix(token, "host:") {
+				host = strings.TrimPrefix(strings.TrimSuffix(token, "\r"), "host:")
 				return method, resource, host, nil
 			}
 		}

cmd/http/listener_test.go

@@ -116,7 +116,7 @@ func getTLSConfig(t *testing.T) *tls.Config {
 	tlsConfig := &tls.Config{
 		PreferServerCipherSuites: true,
 		MinVersion:               tls.VersionTLS12,
-		NextProtos:               []string{"http/1.1", "h2"},
+		NextProtos:               []string{"http/1.1"},
 	}
 	tlsConfig.Certificates = append(tlsConfig.Certificates, tlsCert)
 

cmd/http/server.go

@@ -192,7 +192,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			CipherSuites:             defaultCipherSuites,
 			CurvePreferences:         secureCurves,
 			MinVersion:               tls.VersionTLS12,
-			NextProtos:               []string{"http/1.1", "h2"},
+			NextProtos:               []string{"http/1.1"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/iam.go

@@ -0,0 +1,730 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+const (
+	// IAM configuration directory.
+	iamConfigPrefix = minioConfigPrefix + "/iam"
+
+	// IAM users directory.
+	iamConfigUsersPrefix = iamConfigPrefix + "/users/"
+
+	// IAM policies directory.
+	iamConfigPoliciesPrefix = iamConfigPrefix + "/policies/"
+
+	// IAM sts directory.
+	iamConfigSTSPrefix = iamConfigPrefix + "/sts/"
+
+	// IAM identity file which captures identity credentials.
+	iamIdentityFile = "identity.json"
+
+	// IAM policy file which provides policies for each users.
+	iamPolicyFile = "policy.json"
+)
+
+// IAMSys - config system.
+type IAMSys struct {
+	sync.RWMutex
+	iamUsersMap        map[string]auth.Credentials
+	iamPolicyMap       map[string]string
+	iamCannedPolicyMap map[string]iampolicy.Policy
+}
+
+// Load - load iam.json
+func (sys *IAMSys) Load(objAPI ObjectLayer) error {
+	return sys.Init(objAPI)
+}
+
+// Init - initializes config system from iam.json
+func (sys *IAMSys) Init(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+
+	defer func() {
+		// Refresh IAMSys in background.
+		go func() {
+			ticker := time.NewTicker(globalRefreshIAMInterval)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-globalServiceDoneCh:
+					return
+				case <-ticker.C:
+					sys.refresh(objAPI)
+				}
+			}
+		}()
+	}()
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing IAM needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			// Load IAMSys once during boot.
+			if err := sys.refresh(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for IAM subsystem to be initialized..")
+					continue
+				}
+				return err
+			}
+			return nil
+		}
+	}
+}
+
+// DeleteCannedPolicy - deletes a canned policy.
+func (sys *IAMSys) DeleteCannedPolicy(policyName string) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	if policyName == "" {
+		return errInvalidArgument
+	}
+
+	var err error
+	configFile := pathJoin(iamConfigPoliciesPrefix, policyName, iamPolicyFile)
+	if globalEtcdClient != nil {
+		err = deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
+	} else {
+		err = deleteConfig(context.Background(), objectAPI, configFile)
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	delete(sys.iamCannedPolicyMap, policyName)
+	return err
+}
+
+// ListCannedPolicies - lists all canned policies.
+func (sys *IAMSys) ListCannedPolicies() (map[string][]byte, error) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+
+	var cannedPolicyMap = make(map[string][]byte)
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	for k, v := range sys.iamCannedPolicyMap {
+		data, err := json.Marshal(v)
+		if err != nil {
+			return nil, err
+		}
+		cannedPolicyMap[k] = data
+	}
+
+	return cannedPolicyMap, nil
+}
+
+// SetCannedPolicy - sets a new canned policy.
+func (sys *IAMSys) SetCannedPolicy(policyName string, p iampolicy.Policy) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	if p.IsEmpty() || policyName == "" {
+		return errInvalidArgument
+	}
+
+	configFile := pathJoin(iamConfigPoliciesPrefix, policyName, iamPolicyFile)
+	data, err := json.Marshal(p)
+	if err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+	if err != nil {
+		return err
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.iamCannedPolicyMap[policyName] = p
+
+	return nil
+}
+
+// SetUserPolicy - sets policy to given user name.
+func (sys *IAMSys) SetUserPolicy(accessKey, policyName string) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	if _, ok := sys.iamUsersMap[accessKey]; !ok {
+		return errNoSuchUser
+	}
+
+	if _, ok := sys.iamCannedPolicyMap[policyName]; !ok {
+		return errNoSuchPolicy
+	}
+
+	data, err := json.Marshal(policyName)
+	if err != nil {
+		return err
+	}
+
+	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile)
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+	if err != nil {
+		return err
+	}
+
+	sys.iamPolicyMap[accessKey] = policyName
+	return nil
+}
+
+// DeleteUser - set user credentials.
+func (sys *IAMSys) DeleteUser(accessKey string) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	var err error
+	pFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile)
+	iFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
+	if globalEtcdClient != nil {
+		// It is okay to ingnore errors when deleting policy.json for the user.
+		_ = deleteConfigEtcd(context.Background(), globalEtcdClient, pFile)
+		err = deleteConfigEtcd(context.Background(), globalEtcdClient, iFile)
+	} else {
+		// It is okay to ingnore errors when deleting policy.json for the user.
+		_ = deleteConfig(context.Background(), objectAPI, pFile)
+		err = deleteConfig(context.Background(), objectAPI, iFile)
+	}
+
+	//
+	switch err.(type) {
+	case ObjectNotFound:
+		err = errNoSuchUser
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	delete(sys.iamUsersMap, accessKey)
+	delete(sys.iamPolicyMap, accessKey)
+
+	return err
+}
+
+// SetTempUser - set temporary user credentials, these credentials have an expiry.
+func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyName string) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	// If OPA is not set we honor any policy claims for this
+	// temporary user which match with pre-configured canned
+	// policies for this server.
+	if globalPolicyOPA == nil && policyName != "" {
+		p, ok := sys.iamCannedPolicyMap[policyName]
+		if !ok {
+			return errInvalidArgument
+		}
+		if p.IsEmpty() {
+			delete(sys.iamPolicyMap, accessKey)
+			return nil
+		}
+
+		data, err := json.Marshal(policyName)
+		if err != nil {
+			return err
+		}
+
+		configFile := pathJoin(iamConfigSTSPrefix, accessKey, iamPolicyFile)
+		if globalEtcdClient != nil {
+			err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+		} else {
+			err = saveConfig(context.Background(), objectAPI, configFile, data)
+		}
+		if err != nil {
+			return err
+		}
+
+		sys.iamPolicyMap[accessKey] = policyName
+	}
+
+	configFile := pathJoin(iamConfigSTSPrefix, accessKey, iamIdentityFile)
+	data, err := json.Marshal(cred)
+	if err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	sys.iamUsersMap[accessKey] = cred
+	return nil
+}
+
+// ListUsers - list all users.
+func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+
+	var users = make(map[string]madmin.UserInfo)
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	for k, v := range sys.iamUsersMap {
+		users[k] = madmin.UserInfo{
+			PolicyName: sys.iamPolicyMap[k],
+			Status:     madmin.AccountStatus(v.Status),
+		}
+	}
+
+	return users, nil
+}
+
+// SetUserStatus - sets current user status, supports disabled or enabled.
+func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	if status != madmin.AccountEnabled && status != madmin.AccountDisabled {
+		return errInvalidArgument
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	cred, ok := sys.iamUsersMap[accessKey]
+	if !ok {
+		return errNoSuchUser
+	}
+
+	uinfo := madmin.UserInfo{
+		SecretKey: cred.SecretKey,
+		Status:    status,
+	}
+
+	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
+	data, err := json.Marshal(uinfo)
+	if err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	sys.iamUsersMap[accessKey] = auth.Credentials{
+		AccessKey: accessKey,
+		SecretKey: uinfo.SecretKey,
+		Status:    string(uinfo.Status),
+	}
+
+	return nil
+}
+
+// SetUser - set user credentials.
+func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
+	data, err := json.Marshal(uinfo)
+	if err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.iamUsersMap[accessKey] = auth.Credentials{
+		AccessKey: accessKey,
+		SecretKey: uinfo.SecretKey,
+		Status:    string(uinfo.Status),
+	}
+
+	return nil
+}
+
+// GetUser - get user credentials
+func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
+	sys.RLock()
+	defer sys.RUnlock()
+
+	cred, ok = sys.iamUsersMap[accessKey]
+	return cred, ok && cred.IsValid()
+}
+
+// IsAllowed - checks given policy args is allowed to continue the Rest API.
+func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
+	sys.RLock()
+	defer sys.RUnlock()
+
+	// If opa is configured, use OPA always.
+	if globalPolicyOPA != nil {
+		return globalPolicyOPA.IsAllowed(args)
+	}
+
+	// If policy is available for given user, check the policy.
+	if name, found := sys.iamPolicyMap[args.AccountName]; found {
+		p, ok := sys.iamCannedPolicyMap[name]
+		return ok && p.IsAllowed(args)
+	}
+
+	// As policy is not available and OPA is not configured, return the owner value.
+	return args.IsOwner
+}
+
+var defaultContextTimeout = 5 * time.Minute
+
+// Similar to reloadUsers but updates users, policies maps from etcd server,
+func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policyMap map[string]string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+	defer cancel()
+	if err != nil {
+		return err
+	}
+	// No users are created yet.
+	if r.Count == 0 {
+		return nil
+	}
+
+	users := set.NewStringSet()
+	for _, kv := range r.Kvs {
+		// Extract user by stripping off the `prefix` value as suffix,
+		// then strip off the remaining basename to obtain the prefix
+		// value, usually in the following form.
+		//
+		//  key := "config/iam/users/newuser/identity.json"
+		//  prefix := "config/iam/users/"
+		//  v := trim(trim(key, prefix), base(key)) == "newuser"
+		//
+		user := path.Clean(strings.TrimSuffix(strings.TrimPrefix(string(kv.Key), prefix), path.Base(string(kv.Key))))
+		if !users.Contains(user) {
+			users.Add(user)
+		}
+	}
+
+	// Reload config and policies for all users.
+	for _, user := range users.ToSlice() {
+		idFile := pathJoin(prefix, user, iamIdentityFile)
+		pFile := pathJoin(prefix, user, iamPolicyFile)
+		cdata, cerr := readConfigEtcd(ctx, globalEtcdClient, idFile)
+		pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile)
+		if cerr != nil && cerr != errConfigNotFound {
+			return cerr
+		}
+		if perr != nil && perr != errConfigNotFound {
+			return perr
+		}
+		if cerr == errConfigNotFound && perr == errConfigNotFound {
+			continue
+		}
+		if cerr == nil {
+			var cred auth.Credentials
+			if err = json.Unmarshal(cdata, &cred); err != nil {
+				return err
+			}
+			cred.AccessKey = user
+			if cred.IsExpired() {
+				deleteConfigEtcd(ctx, globalEtcdClient, idFile)
+				deleteConfigEtcd(ctx, globalEtcdClient, pFile)
+				continue
+			}
+			usersMap[cred.AccessKey] = cred
+		}
+		if perr == nil {
+			var policyName string
+			if err = json.Unmarshal(pdata, &policyName); err != nil {
+				return err
+			}
+			policyMap[user] = policyName
+		}
+	}
+	return nil
+}
+
+func reloadEtcdPolicies(prefix string, cannedPolicyMap map[string]iampolicy.Policy) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+	defer cancel()
+	if err != nil {
+		return err
+	}
+	// No users are created yet.
+	if r.Count == 0 {
+		return nil
+	}
+
+	policies := set.NewStringSet()
+	for _, kv := range r.Kvs {
+		// Extract policy by stripping off the `prefix` value as suffix,
+		// then strip off the remaining basename to obtain the prefix
+		// value, usually in the following form.
+		//
+		//  key := "config/iam/policies/newpolicy/identity.json"
+		//  prefix := "config/iam/policies/"
+		//  v := trim(trim(key, prefix), base(key)) == "newpolicy"
+		//
+		policyName := path.Clean(strings.TrimSuffix(strings.TrimPrefix(string(kv.Key), prefix), path.Base(string(kv.Key))))
+		if !policies.Contains(policyName) {
+			policies.Add(policyName)
+		}
+	}
+
+	// Reload config and policies for all policys.
+	for _, policyName := range policies.ToSlice() {
+		pFile := pathJoin(prefix, policyName, iamPolicyFile)
+		pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile)
+		if perr != nil {
+			return perr
+		}
+		var p iampolicy.Policy
+		if err = json.Unmarshal(pdata, &p); err != nil {
+			return err
+		}
+		cannedPolicyMap[policyName] = p
+	}
+	return nil
+}
+
+func reloadPolicies(objectAPI ObjectLayer, prefix string, cannedPolicyMap map[string]iampolicy.Policy) error {
+	marker := ""
+	for {
+		var lo ListObjectsInfo
+		var err error
+		lo, err = objectAPI.ListObjects(context.Background(), minioMetaBucket, prefix, marker, "/", 1000)
+		if err != nil {
+			return err
+		}
+		marker = lo.NextMarker
+		for _, prefix := range lo.Prefixes {
+			pFile := pathJoin(prefix, iamPolicyFile)
+			pdata, perr := readConfig(context.Background(), objectAPI, pFile)
+			if perr != nil {
+				return perr
+			}
+			var p iampolicy.Policy
+			if err = json.Unmarshal(pdata, &p); err != nil {
+				return err
+			}
+			cannedPolicyMap[path.Base(prefix)] = p
+		}
+		if !lo.IsTruncated {
+			break
+		}
+	}
+	return nil
+
+}
+
+// reloadUsers reads an updates users, policies from object layer into user and policy maps.
+func reloadUsers(objectAPI ObjectLayer, prefix string, usersMap map[string]auth.Credentials, policyMap map[string]string) error {
+	marker := ""
+	for {
+		var lo ListObjectsInfo
+		var err error
+		lo, err = objectAPI.ListObjects(context.Background(), minioMetaBucket, prefix, marker, "/", 1000)
+		if err != nil {
+			return err
+		}
+		marker = lo.NextMarker
+		for _, prefix := range lo.Prefixes {
+			idFile := pathJoin(prefix, iamIdentityFile)
+			pFile := pathJoin(prefix, iamPolicyFile)
+			cdata, cerr := readConfig(context.Background(), objectAPI, idFile)
+			pdata, perr := readConfig(context.Background(), objectAPI, pFile)
+			if cerr != nil && cerr != errConfigNotFound {
+				return cerr
+			}
+			if perr != nil && perr != errConfigNotFound {
+				return perr
+			}
+			if cerr == errConfigNotFound && perr == errConfigNotFound {
+				continue
+			}
+			if cerr == nil {
+				var cred auth.Credentials
+				if err = json.Unmarshal(cdata, &cred); err != nil {
+					return err
+				}
+				cred.AccessKey = path.Base(prefix)
+				if cred.IsExpired() {
+					// Delete expired identity.
+					objectAPI.DeleteObject(context.Background(), minioMetaBucket, idFile)
+					// Delete expired identity policy.
+					objectAPI.DeleteObject(context.Background(), minioMetaBucket, pFile)
+					continue
+				}
+				usersMap[cred.AccessKey] = cred
+			}
+			if perr == nil {
+				var policyName string
+				if err = json.Unmarshal(pdata, &policyName); err != nil {
+					return err
+				}
+				policyMap[path.Base(prefix)] = policyName
+			}
+		}
+		if !lo.IsTruncated {
+			break
+		}
+	}
+	return nil
+}
+
+// Set default canned policies only if not already overridden by users.
+func setDefaultCannedPolicies(policies map[string]iampolicy.Policy) {
+	_, ok := policies["writeonly"]
+	if !ok {
+		policies["writeonly"] = iampolicy.WriteOnly
+	}
+	_, ok = policies["readonly"]
+	if !ok {
+		policies["readonly"] = iampolicy.ReadOnly
+	}
+	_, ok = policies["readwrite"]
+	if !ok {
+		policies["readwrite"] = iampolicy.ReadWrite
+	}
+}
+
+// Refresh IAMSys.
+func (sys *IAMSys) refresh(objAPI ObjectLayer) error {
+	iamUsersMap := make(map[string]auth.Credentials)
+	iamPolicyMap := make(map[string]string)
+	iamCannedPolicyMap := make(map[string]iampolicy.Policy)
+
+	if globalEtcdClient != nil {
+		if err := reloadEtcdPolicies(iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadEtcdUsers(iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadEtcdUsers(iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+	} else {
+		if err := reloadPolicies(objAPI, iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+	}
+
+	// Sets default canned policies, if none set.
+	setDefaultCannedPolicies(iamCannedPolicyMap)
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.iamUsersMap = iamUsersMap
+	sys.iamPolicyMap = iamPolicyMap
+	sys.iamCannedPolicyMap = iamCannedPolicyMap
+
+	return nil
+}
+
+// NewIAMSys - creates new config system object.
+func NewIAMSys() *IAMSys {
+	return &IAMSys{
+		iamUsersMap:        make(map[string]auth.Credentials),
+		iamPolicyMap:       make(map[string]string),
+		iamCannedPolicyMap: make(map[string]iampolicy.Policy),
+	}
+}

cmd/jwt.go

@@ -49,7 +49,33 @@ var (
 	errNoAuthToken          = errors.New("JWT token missing")
 )
 
-func authenticateJWT(accessKey, secretKey string, expiry time.Duration) (string, error) {
+func authenticateJWTUsers(accessKey, secretKey string, expiry time.Duration) (string, error) {
+	passedCredential, err := auth.CreateCredentials(accessKey, secretKey)
+	if err != nil {
+		return "", err
+	}
+
+	serverCred := globalServerConfig.GetCredential()
+	if serverCred.AccessKey != passedCredential.AccessKey {
+		var ok bool
+		serverCred, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return "", errInvalidAccessKeyID
+		}
+	}
+
+	if !serverCred.Equal(passedCredential) {
+		return "", errAuthentication
+	}
+
+	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{
+		ExpiresAt: UTCNow().Add(expiry).Unix(),
+		Subject:   accessKey,
+	})
+	return jwt.SignedString([]byte(serverCred.SecretKey))
+}
+
+func authenticateJWTAdmin(accessKey, secretKey string, expiry time.Duration) (string, error) {
 	passedCredential, err := auth.CreateCredentials(accessKey, secretKey)
 	if err != nil {
 		return "", err
@@ -73,68 +99,133 @@ func authenticateJWT(accessKey, secretKey string, expiry time.Duration) (string,
 }
 
 func authenticateNode(accessKey, secretKey string) (string, error) {
-	return authenticateJWT(accessKey, secretKey, defaultInterNodeJWTExpiry)
+	return authenticateJWTAdmin(accessKey, secretKey, defaultInterNodeJWTExpiry)
 }
 
 func authenticateWeb(accessKey, secretKey string) (string, error) {
-	return authenticateJWT(accessKey, secretKey, defaultJWTExpiry)
+	return authenticateJWTUsers(accessKey, secretKey, defaultJWTExpiry)
 }
 
 func authenticateURL(accessKey, secretKey string) (string, error) {
-	return authenticateJWT(accessKey, secretKey, defaultURLJWTExpiry)
+	return authenticateJWTUsers(accessKey, secretKey, defaultURLJWTExpiry)
 }
 
-func keyFuncCallback(jwtToken *jwtgo.Token) (interface{}, error) {
+func stsTokenCallback(jwtToken *jwtgo.Token) (interface{}, error) {
 	if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
 		return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
 	}
 
-	return []byte(globalServerConfig.GetCredential().SecretKey), nil
+	if err := jwtToken.Claims.Valid(); err != nil {
+		return nil, errAuthentication
+	}
+	if claims, ok := jwtToken.Claims.(jwtgo.MapClaims); ok {
+		accessKey, ok := claims["accessKey"].(string)
+		if !ok {
+			return nil, errInvalidAccessKeyID
+		}
+		if accessKey == globalServerConfig.GetCredential().AccessKey {
+			return []byte(globalServerConfig.GetCredential().SecretKey), nil
+		}
+		if globalIAMSys == nil {
+			return nil, errInvalidAccessKeyID
+		}
+		_, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return nil, errInvalidAccessKeyID
+		}
+		return []byte(globalServerConfig.GetCredential().SecretKey), nil
+	}
+	return nil, errAuthentication
 }
 
-func isAuthTokenValid(tokenString string) bool {
-	if tokenString == "" {
-		return false
+// Callback function used for parsing
+func webTokenCallback(jwtToken *jwtgo.Token) (interface{}, error) {
+	if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
+		return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
 	}
-	var claims jwtgo.StandardClaims
-	jwtToken, err := jwtgo.ParseWithClaims(tokenString, &claims, keyFuncCallback)
+
+	if err := jwtToken.Claims.Valid(); err != nil {
+		return nil, errAuthentication
+	}
+
+	if claims, ok := jwtToken.Claims.(*jwtgo.StandardClaims); ok {
+		if claims.Subject == globalServerConfig.GetCredential().AccessKey {
+			return []byte(globalServerConfig.GetCredential().SecretKey), nil
+		}
+		if globalIAMSys == nil {
+			return nil, errInvalidAccessKeyID
+		}
+		cred, ok := globalIAMSys.GetUser(claims.Subject)
+		if !ok {
+			return nil, errInvalidAccessKeyID
+		}
+		return []byte(cred.SecretKey), nil
+	}
+
+	return nil, errAuthentication
+}
+
+func parseJWTWithClaims(tokenString string, claims jwtgo.Claims) (*jwtgo.Token, error) {
+	p := &jwtgo.Parser{
+		SkipClaimsValidation: true,
+	}
+	jwtToken, err := p.ParseWithClaims(tokenString, claims, webTokenCallback)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return false
+		switch e := err.(type) {
+		case *jwtgo.ValidationError:
+			if e.Inner == nil {
+				return nil, errAuthentication
+			}
+			return nil, e.Inner
+		}
+		return nil, errAuthentication
 	}
-	if err = claims.Valid(); err != nil {
-		logger.LogIf(context.Background(), err)
-		return false
-	}
-	return jwtToken.Valid && claims.Subject == globalServerConfig.GetCredential().AccessKey
+	return jwtToken, nil
 }
 
-func isHTTPRequestValid(req *http.Request) bool {
-	return webRequestAuthenticate(req) == nil
+func isAuthTokenValid(token string) bool {
+	_, _, err := webTokenAuthenticate(token)
+	return err == nil
+}
+
+func webTokenAuthenticate(token string) (jwtgo.StandardClaims, bool, error) {
+	var claims = jwtgo.StandardClaims{}
+	if token == "" {
+		return claims, false, errNoAuthToken
+	}
+
+	jwtToken, err := parseJWTWithClaims(token, &claims)
+	if err != nil {
+		return claims, false, err
+	}
+	if !jwtToken.Valid {
+		return claims, false, errAuthentication
+	}
+	owner := claims.Subject == globalServerConfig.GetCredential().AccessKey
+	return claims, owner, nil
 }
 
 // Check if the request is authenticated.
 // Returns nil if the request is authenticated. errNoAuthToken if token missing.
 // Returns errAuthentication for all other errors.
-func webRequestAuthenticate(req *http.Request) error {
-	var claims jwtgo.StandardClaims
-	jwtToken, err := jwtreq.ParseFromRequestWithClaims(req, jwtreq.AuthorizationHeaderExtractor, &claims, keyFuncCallback)
+func webRequestAuthenticate(req *http.Request) (jwtgo.StandardClaims, bool, error) {
+	var claims = jwtgo.StandardClaims{}
+	tokStr, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(req)
 	if err != nil {
 		if err == jwtreq.ErrNoTokenInRequest {
-			return errNoAuthToken
+			return claims, false, errNoAuthToken
 		}
-		return errAuthentication
+		return claims, false, err
 	}
-	if err = claims.Valid(); err != nil {
-		return errAuthentication
-	}
-	if claims.Subject != globalServerConfig.GetCredential().AccessKey {
-		return errInvalidAccessKeyID
+	jwtToken, err := parseJWTWithClaims(tokStr, &claims)
+	if err != nil {
+		return claims, false, err
 	}
 	if !jwtToken.Valid {
-		return errAuthentication
+		return claims, false, errAuthentication
 	}
-	return nil
+	owner := claims.Subject == globalServerConfig.GetCredential().AccessKey
+	return claims, owner, nil
 }
 
 func newAuthToken() string {

cmd/jwt_test.go

@@ -142,7 +142,7 @@ func TestWebRequestAuthenticate(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		gotErr := webRequestAuthenticate(testCase.req)
+		_, _, gotErr := webRequestAuthenticate(testCase.req)
 		if testCase.expectedErr != gotErr {
 			t.Errorf("Test %d, expected err %s, got %s", i+1, testCase.expectedErr, gotErr)
 		}

cmd/logger/audit-logger.go

@@ -0,0 +1,97 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Represents the current version of audit log structure.
+const auditLogVersion = "1"
+
+// AuditEntry - audit entry logs.
+type AuditEntry struct {
+	Version      string            `json:"version"`
+	DeploymentID string            `json:"deploymentid,omitempty"`
+	Time         string            `json:"time"`
+	API          *api              `json:"api,omitempty"`
+	RemoteHost   string            `json:"remotehost,omitempty"`
+	RequestID    string            `json:"requestID,omitempty"`
+	UserAgent    string            `json:"userAgent,omitempty"`
+	ReqQuery     map[string]string `json:"requestQuery,omitempty"`
+	ReqHeader    map[string]string `json:"requestHeader,omitempty"`
+	RespHeader   map[string]string `json:"responseHeader,omitempty"`
+}
+
+// AuditTargets is the list of enabled audit loggers
+var AuditTargets = []LoggingTarget{}
+
+// AddAuditTarget adds a new audit logger target to the
+// list of enabled loggers
+func AddAuditTarget(t LoggingTarget) {
+	AuditTargets = append(AuditTargets, t)
+}
+
+// AuditLog - logs audit logs to all targets.
+func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if Disable {
+		return
+	}
+
+	req := GetReqInfo(ctx)
+	if req == nil {
+		req = &ReqInfo{API: "SYSTEM"}
+	}
+
+	reqQuery := make(map[string]string)
+	for k, v := range r.URL.Query() {
+		reqQuery[k] = strings.Join(v, ",")
+	}
+	reqHeader := make(map[string]string)
+	for k, v := range r.Header {
+		reqHeader[k] = strings.Join(v, ",")
+	}
+	respHeader := make(map[string]string)
+	for k, v := range w.Header() {
+		respHeader[k] = strings.Join(v, ",")
+	}
+
+	// Send audit logs only to http targets.
+	for _, t := range AuditTargets {
+		t.send(AuditEntry{
+			Version:      auditLogVersion,
+			DeploymentID: deploymentID,
+			RemoteHost:   req.RemoteHost,
+			RequestID:    req.RequestID,
+			UserAgent:    req.UserAgent,
+			Time:         time.Now().UTC().Format(time.RFC3339Nano),
+			API: &api{
+				Name: req.API,
+				Args: &args{
+					Bucket: req.BucketName,
+					Object: req.ObjectName,
+				},
+			},
+			ReqQuery:   reqQuery,
+			ReqHeader:  reqHeader,
+			RespHeader: respHeader,
+		})
+	}
+}

cmd/logger/console.go

@@ -27,7 +27,11 @@ import (
 // in plain or json format to the standard output.
 type ConsoleTarget struct{}
 
-func (c *ConsoleTarget) send(entry logEntry) error {
+func (c *ConsoleTarget) send(e interface{}) error {
+	entry, ok := e.(logEntry)
+	if !ok {
+		return fmt.Errorf("Uexpected log entry structure %#v", e)
+	}
 	if jsonFlag {
 		logJSON, err := json.Marshal(&entry)
 		if err != nil {

cmd/logger/http.go

@@ -32,7 +32,8 @@ import (
 // is returned to the caller.
 type HTTPTarget struct {
 	// Channel of log entries
-	logCh chan logEntry
+	logCh chan interface{}
+
 	// HTTP(s) endpoint
 	endpoint string
 	client   http.Client
@@ -72,14 +73,14 @@ func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
 		client: http.Client{
 			Transport: transport,
 		},
-		logCh: make(chan logEntry, 10000),
+		logCh: make(chan interface{}, 10000),
 	}
 
 	h.startHTTPLogger()
 	return &h
 }
 
-func (h *HTTPTarget) send(entry logEntry) error {
+func (h *HTTPTarget) send(entry interface{}) error {
 	select {
 	case h.logCh <- entry:
 	default:

cmd/logger/logger.go

@@ -116,9 +116,11 @@ type traceEntry struct {
 	Source    []string          `json:"source,omitempty"`
 	Variables map[string]string `json:"variables,omitempty"`
 }
+
 type args struct {
-	Bucket string `json:"bucket,omitempty"`
-	Object string `json:"object,omitempty"`
+	Bucket   string            `json:"bucket,omitempty"`
+	Object   string            `json:"object,omitempty"`
+	Metadata map[string]string `json:"metadata,omitempty"`
 }
 
 type api struct {
@@ -319,7 +321,7 @@ func logIf(ctx context.Context, err error) {
 	}
 
 	// Get full stack trace
-	trace := getTrace(2)
+	trace := getTrace(3)
 
 	// Get the cause for the Error
 	message := err.Error()
@@ -450,10 +452,9 @@ func (f fatalMsg) pretty(msg string, args ...interface{}) {
 	os.Exit(1)
 }
 
-var info infoMsg
+type infoMsg struct{}
 
-type infoMsg struct {
-}
+var info infoMsg
 
 func (i infoMsg) json(msg string, args ...interface{}) {
 	logJSON, err := json.Marshal(&logEntry{

cmd/logger/targets.go

@@ -20,7 +20,7 @@ package logger
 // a single log entry and send it to the log target
 //   e.g. send the log to a http server
 type LoggingTarget interface {
-	send(entry logEntry) error
+	send(entry interface{}) error
 }
 
 // Targets is the set of enabled loggers

cmd/namespace-lock_test.go

@@ -218,7 +218,8 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
 		// Try to claim lock again.
 		anotherLock := globalNSMutex.NewNSLock("bucket", "object")
 		if anotherLock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
-			t.Fatalf("Failed to get lock")
+			t.Errorf("Failed to get lock")
+			return
 		}
 		// And signal success.
 		ch <- struct{}{}

cmd/notification.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"encoding/xml"
@@ -241,20 +242,19 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
+	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
 		return err
 	}
-	defer objLock.Unlock()
+	defer objLock.RUnlock()
 
-	reader, e := readConfig(ctx, objAPI, configFile)
+	configData, e := readConfig(ctx, objAPI, configFile)
 	if e != nil && !IsErrIgnored(e, errDiskNotFound, errConfigNotFound) {
 		return e
 	}
 
 	listenerList := []ListenBucketNotificationArgs{}
-	if reader != nil {
-		err := json.NewDecoder(reader).Decode(&listenerList)
-		if err != nil {
+	if configData != nil {
+		if err := json.Unmarshal(configData, &listenerList); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
@@ -265,7 +265,6 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 		return nil
 	}
 
-	activeListenerList := []ListenBucketNotificationArgs{}
 	for _, args := range listenerList {
 		found, err := isLocalHost(args.Addr.Name)
 		if err != nil {
@@ -301,16 +300,31 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 			logger.LogIf(ctx, err)
 			return err
 		}
-		activeListenerList = append(activeListenerList, args)
 	}
 
-	data, err := json.Marshal(activeListenerList)
+	return nil
+}
+
+func (sys *NotificationSys) refresh(objAPI ObjectLayer) error {
+	buckets, err := objAPI.ListBuckets(context.Background())
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return err
 	}
-
-	return saveConfig(objAPI, configFile, data)
+	for _, bucket := range buckets {
+		ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name})
+		config, err := readNotificationConfig(ctx, objAPI, bucket.Name)
+		if err != nil && err != errNoSuchNotifications {
+			return err
+		}
+		if err == errNoSuchNotifications {
+			continue
+		}
+		sys.AddRulesMap(bucket.Name, config.ToRulesMap())
+		if err = sys.initListeners(ctx, objAPI, bucket.Name); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // Init - initializes notification system from notification.xml and listener.json of all buckets.
@@ -319,28 +333,29 @@ func (sys *NotificationSys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	buckets, err := objAPI.ListBuckets(context.Background())
-	if err != nil {
-		return err
-	}
+	doneCh := make(chan struct{})
+	defer close(doneCh)
 
-	for _, bucket := range buckets {
-		ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name})
-		config, err := readNotificationConfig(ctx, objAPI, bucket.Name)
-		if err != nil {
-			if !IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+	// Initializing notification needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			if err := sys.refresh(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for notification subsystem to be initialized..")
+					continue
+				}
 				return err
 			}
-		} else {
-			sys.AddRulesMap(bucket.Name, config.ToRulesMap())
-		}
-
-		if err = sys.initListeners(ctx, objAPI, bucket.Name); err != nil {
-			return err
+			return nil
 		}
 	}
-
-	return nil
 }
 
 // AddRulesMap - adds rules map for bucket name.
@@ -433,6 +448,7 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
 	sys.RLock()
 	targetIDSet := sys.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name)
 	sys.RUnlock()
+
 	if len(targetIDSet) == 0 {
 		return nil
 	}
@@ -478,12 +494,11 @@ func (args eventArgs) ToEvent() event.Event {
 		return fmt.Sprintf("%s://%s:%s", getURLScheme(globalIsSSL), host, globalMinioPort)
 	}
 
-	creds := globalServerConfig.GetCredential()
 	eventTime := UTCNow()
 	uniqueID := fmt.Sprintf("%X", eventTime.UnixNano())
 
 	respElements := map[string]string{
-		"x-amz-request-id":        uniqueID,
+		"x-amz-request-id":        args.RespElements["requestId"],
 		"x-minio-origin-endpoint": getOriginEndpoint(), // Minio specific custom elements.
 	}
 	if args.RespElements["content-length"] != "" {
@@ -492,10 +507,10 @@ func (args eventArgs) ToEvent() event.Event {
 	newEvent := event.Event{
 		EventVersion:      "2.0",
 		EventSource:       "minio:s3",
-		AwsRegion:         globalServerConfig.GetRegion(),
+		AwsRegion:         args.ReqParams["region"],
 		EventTime:         eventTime.Format(event.AMZTimeFormat),
 		EventName:         args.EventName,
-		UserIdentity:      event.Identity{creds.AccessKey},
+		UserIdentity:      event.Identity{args.ReqParams["accessKey"]},
 		RequestParameters: args.ReqParams,
 		ResponseElements:  respElements,
 		S3: event.Metadata{
@@ -503,7 +518,7 @@ func (args eventArgs) ToEvent() event.Event {
 			ConfigurationID: "Config",
 			Bucket: event.Bucket{
 				Name:          args.BucketName,
-				OwnerIdentity: event.Identity{creds.AccessKey},
+				OwnerIdentity: event.Identity{args.ReqParams["accessKey"]},
 				ARN:           policy.ResourceARNPrefix + args.BucketName,
 			},
 			Object: event.Object{
@@ -550,7 +565,7 @@ func sendEvent(args eventArgs) {
 func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) (*event.Config, error) {
 	// Construct path to notification.xml for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
-	reader, err := readConfig(ctx, objAPI, configFile)
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
 		if err == errConfigNotFound {
 			err = errNoSuchNotifications
@@ -559,19 +574,19 @@ func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName
 		return nil, err
 	}
 
-	config, err := event.ParseConfig(reader, globalServerConfig.GetRegion(), globalNotificationSys.targetList)
+	config, err := event.ParseConfig(bytes.NewReader(configData), globalServerConfig.GetRegion(), globalNotificationSys.targetList)
 	logger.LogIf(ctx, err)
 	return config, err
 }
 
-func saveNotificationConfig(objAPI ObjectLayer, bucketName string, config *event.Config) error {
+func saveNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, config *event.Config) error {
 	data, err := xml.Marshal(config)
 	if err != nil {
 		return err
 	}
 
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
-	return saveConfig(objAPI, configFile, data)
+	return saveConfig(ctx, objAPI, configFile, data)
 }
 
 // SaveListener - saves HTTP client currently listening for events to listener.json.
@@ -596,14 +611,14 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
 	}
 	defer objLock.Unlock()
 
-	reader, err := readConfig(ctx, objAPI, configFile)
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil && !IsErrIgnored(err, errDiskNotFound, errConfigNotFound) {
 		return err
 	}
 
 	listenerList := []ListenBucketNotificationArgs{}
-	if reader != nil {
-		if err = json.NewDecoder(reader).Decode(&listenerList); err != nil {
+	if configData != nil {
+		if err = json.Unmarshal(configData, &listenerList); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
@@ -622,7 +637,7 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
 		return err
 	}
 
-	return saveConfig(objAPI, configFile, data)
+	return saveConfig(ctx, objAPI, configFile, data)
 }
 
 // RemoveListener - removes HTTP client currently listening for events from listener.json.
@@ -647,14 +662,14 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
 	}
 	defer objLock.Unlock()
 
-	reader, err := readConfig(ctx, objAPI, configFile)
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil && !IsErrIgnored(err, errDiskNotFound, errConfigNotFound) {
 		return err
 	}
 
 	listenerList := []ListenBucketNotificationArgs{}
-	if reader != nil {
-		if err = json.NewDecoder(reader).Decode(&listenerList); err != nil {
+	if configData != nil {
+		if err = json.Unmarshal(configData, &listenerList); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
@@ -681,5 +696,5 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
 		return err
 	}
 
-	return saveConfig(objAPI, configFile, data)
+	return saveConfig(ctx, objAPI, configFile, data)
 }

cmd/object-api-datatypes.go

@@ -109,8 +109,12 @@ type ObjectInfo struct {
 	Writer       io.WriteCloser `json:"-"`
 	Reader       *hash.Reader   `json:"-"`
 	metadataOnly bool
+
 	// Date and time when the object was last accessed.
 	AccTime time.Time
+
+	// backendType indicates which backend filled this structure
+	backendType BackendType
 }
 
 // ListPartsInfo - represents list of all parts.

cmd/object-api-errors.go

@@ -378,26 +378,8 @@ func (e BackendDown) Error() string {
 	return "Backend down"
 }
 
-// isErrIncompleteBody - Check if error type is IncompleteBody.
-func isErrIncompleteBody(err error) bool {
-	_, ok := err.(IncompleteBody)
-	return ok
-}
-
 // isErrObjectNotFound - Check if error type is ObjectNotFound.
 func isErrObjectNotFound(err error) bool {
 	_, ok := err.(ObjectNotFound)
 	return ok
 }
-
-// isInsufficientReadQuorum - Check if error type is InsufficientReadQuorum.
-func isInsufficientReadQuorum(err error) bool {
-	_, ok := err.(InsufficientReadQuorum)
-	return ok
-}
-
-// isInsufficientWriteQuorum - Check if error type is InsufficientWriteQuorum.
-func isInsufficientWriteQuorum(err error) bool {
-	_, ok := err.(InsufficientWriteQuorum)
-	return ok
-}

cmd/object-api-putobject_test.go

@@ -159,6 +159,16 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 		// Test case 30
 		// valid data with X-Amz-Meta- meta
 		{bucket, object, data, map[string]string{"X-Amz-Meta-AppID": "a42"}, "", int64(len(data)), getMD5Hash(data), nil},
+
+		// Test case 31
+		// Put an empty object with a trailing slash
+		{bucket, "emptydir/", []byte{}, nil, "", 0, getMD5Hash([]byte{}), nil},
+		// Test case 32
+		// Put an object inside the empty directory
+		{bucket, "emptydir/" + object, data, nil, "", int64(len(data)), getMD5Hash(data), nil},
+		// Test case 33
+		// Put the empty object with a trailing slash again (refer to Test case 31), this needs to succeed
+		{bucket, "emptydir/", []byte{}, nil, "", 0, getMD5Hash([]byte{}), nil},
 	}
 
 	for i, testCase := range testCases {
@@ -171,7 +181,7 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 		}
 		// Failed as expected, but does it fail for the expected reason.
 		if actualErr != nil && actualErr != testCase.expectedError {
-			t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error())
+			t.Errorf("Test %d: %s: Expected to fail with error \"%v\", but instead failed with error \"%v\" instead.", i+1, instanceType, testCase.expectedError, actualErr)
 		}
 		// Test passes as expected, but the output values are verified for correctness here.
 		if actualErr == nil {

cmd/object-handlers.go

@@ -33,6 +33,7 @@ import (
 
 	snappy "github.com/golang/snappy"
 	"github.com/gorilla/mux"
+	"github.com/klauspost/readahead"
 	miniogo "github.com/minio/minio-go"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
@@ -78,13 +79,18 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
 func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SelectObject")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	// Fetch object stat info.
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-
+	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
+		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		return
+	}
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -138,8 +144,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		writeErrorResponse(w, ErrEmptyRequestBody, r.URL)
 		return
 	}
-
-	var selectReq ObjectSelectRequest
+	var selectReq s3select.ObjectSelectRequest
 	if err := xmlDecoder(r.Body, &selectReq, r.ContentLength); err != nil {
 		writeErrorResponse(w, ErrMalformedXML, r.URL)
 		return
@@ -168,22 +173,21 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 
 	objInfo := gr.ObjInfo
 
-	if selectReq.InputSerialization.CompressionType == SelectCompressionGZIP {
+	if selectReq.InputSerialization.CompressionType == s3select.SelectCompressionGZIP {
 		if !strings.Contains(objInfo.ContentType, "gzip") {
 			writeErrorResponse(w, ErrInvalidDataSource, r.URL)
 			return
 		}
 	}
-	if selectReq.InputSerialization.CompressionType == SelectCompressionBZIP {
+	if selectReq.InputSerialization.CompressionType == s3select.SelectCompressionBZIP {
 		if !strings.Contains(objInfo.ContentType, "bzip") {
 			writeErrorResponse(w, ErrInvalidDataSource, r.URL)
 			return
 		}
 	}
-	if selectReq.InputSerialization.CompressionType == SelectCompressionNONE ||
-		selectReq.InputSerialization.CompressionType == "" {
-		selectReq.InputSerialization.CompressionType = SelectCompressionNONE
-		if !strings.Contains(objInfo.ContentType, "text/csv") {
+	if selectReq.InputSerialization.CompressionType == "" {
+		selectReq.InputSerialization.CompressionType = s3select.SelectCompressionNONE
+		if !strings.Contains(objInfo.ContentType, "text/csv") && !strings.Contains(objInfo.ContentType, "application/json") {
 			writeErrorResponse(w, ErrInvalidDataSource, r.URL)
 			return
 		}
@@ -196,28 +200,45 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		writeErrorResponse(w, ErrExpressionTooLong, r.URL)
 		return
 	}
-	if selectReq.InputSerialization.CSV == nil || selectReq.OutputSerialization.CSV == nil {
+	if selectReq.InputSerialization.CSV == nil && selectReq.InputSerialization.JSON == nil {
 		writeErrorResponse(w, ErrInvalidRequestParameter, r.URL)
 		return
 	}
-	if selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoUse &&
-		selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoNone &&
-		selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoIgnore &&
-		selectReq.InputSerialization.CSV.FileHeaderInfo != "" {
-		writeErrorResponse(w, ErrInvalidFileHeaderInfo, r.URL)
-		return
-	}
-	if selectReq.OutputSerialization.CSV.QuoteFields != CSVQuoteFieldsAlways &&
-		selectReq.OutputSerialization.CSV.QuoteFields != CSVQuoteFieldsAsNeeded &&
-		selectReq.OutputSerialization.CSV.QuoteFields != "" {
-		writeErrorResponse(w, ErrInvalidQuoteFields, r.URL)
-		return
-	}
-	if len(selectReq.InputSerialization.CSV.RecordDelimiter) > 2 {
+	if selectReq.OutputSerialization.CSV == nil && selectReq.OutputSerialization.JSON == nil {
 		writeErrorResponse(w, ErrInvalidRequestParameter, r.URL)
 		return
 	}
 
+	if selectReq.InputSerialization.CSV != nil {
+		if selectReq.InputSerialization.CSV.FileHeaderInfo != s3select.CSVFileHeaderInfoUse &&
+			selectReq.InputSerialization.CSV.FileHeaderInfo != s3select.CSVFileHeaderInfoNone &&
+			selectReq.InputSerialization.CSV.FileHeaderInfo != s3select.CSVFileHeaderInfoIgnore &&
+			selectReq.InputSerialization.CSV.FileHeaderInfo != "" {
+			writeErrorResponse(w, ErrInvalidFileHeaderInfo, r.URL)
+			return
+		}
+		if selectReq.OutputSerialization.CSV.QuoteFields != s3select.CSVQuoteFieldsAlways &&
+			selectReq.OutputSerialization.CSV.QuoteFields != s3select.CSVQuoteFieldsAsNeeded &&
+			selectReq.OutputSerialization.CSV.QuoteFields != "" {
+			writeErrorResponse(w, ErrInvalidQuoteFields, r.URL)
+			return
+		}
+		if len(selectReq.InputSerialization.CSV.RecordDelimiter) > 2 {
+			writeErrorResponse(w, ErrInvalidRequestParameter, r.URL)
+			return
+		}
+
+	}
+	if selectReq.InputSerialization.JSON != nil {
+		if selectReq.InputSerialization.JSON.Type != s3select.JSONTypeDocument &&
+			selectReq.InputSerialization.JSON.Type != s3select.JSONLinesType &&
+			selectReq.InputSerialization.JSON.Type != "" {
+			writeErrorResponse(w, ErrInvalidJSONType, r.URL)
+			return
+		}
+
+	}
+
 	// Set encryption response headers
 	if objectAPI.IsEncryptionSupported() {
 		if crypto.IsEncrypted(objInfo.UserDefined) {
@@ -231,45 +252,42 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		}
 	}
 
-	//s3select //Options
-	if selectReq.OutputSerialization.CSV.FieldDelimiter == "" {
-		selectReq.OutputSerialization.CSV.FieldDelimiter = ","
+	reader := readahead.NewReader(gr)
+	defer reader.Close()
+
+	s3s, err := s3select.New(reader, objInfo.GetActualSize(), selectReq)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
 	}
-	if selectReq.InputSerialization.CSV.FileHeaderInfo == "" {
-		selectReq.InputSerialization.CSV.FileHeaderInfo = CSVFileHeaderInfoNone
+
+	// Parses the select query and checks for an error
+	_, _, _, _, _, _, err = s3select.ParseSelect(s3s)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
 	}
-	if selectReq.InputSerialization.CSV.RecordDelimiter == "" {
-		selectReq.InputSerialization.CSV.RecordDelimiter = "\n"
-	}
-	if selectReq.InputSerialization.CSV != nil {
-		options := &s3select.Options{
-			HasHeader:            selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoNone,
-			RecordDelimiter:      selectReq.InputSerialization.CSV.RecordDelimiter,
-			FieldDelimiter:       selectReq.InputSerialization.CSV.FieldDelimiter,
-			Comments:             selectReq.InputSerialization.CSV.Comments,
-			Name:                 "S3Object", // Default table name for all objects
-			ReadFrom:             gr,
-			Compressed:           string(selectReq.InputSerialization.CompressionType),
-			Expression:           selectReq.Expression,
-			OutputFieldDelimiter: selectReq.OutputSerialization.CSV.FieldDelimiter,
-			StreamSize:           objInfo.Size,
-			HeaderOpt:            selectReq.InputSerialization.CSV.FileHeaderInfo == CSVFileHeaderInfoUse,
-			Progress:             selectReq.RequestProgress.Enabled,
-		}
-		s3s, err := s3select.NewInput(options)
-		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-			return
-		}
-		_, _, _, _, _, _, err = s3s.ParseSelect(selectReq.Expression)
-		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-			return
-		}
-		if err = s3s.Execute(w); err != nil {
-			logger.LogIf(ctx, err)
-		}
+
+	// Executes the query on data-set
+	s3select.Execute(w, s3s)
+
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
+	if err != nil {
+		host, port = "", ""
 	}
+
+	// Notify object accessed via a GET request.
+	sendEvent(eventArgs{
+		EventName:    event.ObjectAccessedGet,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
+	})
 }
 
 // GetObjectHandler - GET Object
@@ -279,6 +297,8 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObject")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -361,6 +381,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 	defer gr.Close()
+
 	objInfo := gr.ObjInfo
 
 	if objectAPI.IsEncryptionSupported() {
@@ -442,6 +463,8 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadObject")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponseHeadersOnly(w, ErrServerNotInitialized)
@@ -531,6 +554,11 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 			case crypto.S3.IsEncrypted(objInfo.UserDefined):
 				w.Header().Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 			case crypto.SSEC.IsEncrypted(objInfo.UserDefined):
+				// Validate the SSE-C Key set in the header.
+				if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil {
+					writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
+					return
+				}
 				w.Header().Set(crypto.SSECAlgorithm, r.Header.Get(crypto.SSECAlgorithm))
 				w.Header().Set(crypto.SSECKeyMD5, r.Header.Get(crypto.SSECKeyMD5))
 			}
@@ -544,7 +572,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// Set standard object headers.
 	if hErr := setObjectHeaders(w, objInfo, rs); hErr != nil {
-		writeErrorResponse(w, toAPIErrorCode(hErr), r.URL)
+		writeErrorResponseHeadersOnly(w, toAPIErrorCode(hErr))
 		return
 	}
 
@@ -615,12 +643,14 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
 func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObject")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -650,6 +680,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	// Check if metadata directive is valid.
 	if !isMetadataDirectiveValid(r.Header) {
 		writeErrorResponse(w, ErrInvalidMetadataDirective, r.URL)
@@ -673,30 +708,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		getObjectNInfo = api.CacheAPI().GetObjectNInfo
 	}
 
-	// Get request range.
-	var rs *HTTPRangeSpec
-	rangeHeader := r.Header.Get("x-amz-copy-source-range")
-	if rangeHeader != "" {
-		var parseRangeErr error
-		if rs, parseRangeErr = parseRequestRangeSpec(rangeHeader); parseRangeErr != nil {
-			// Handle only errInvalidRange. Ignore other
-			// parse error and treat it as regular Get
-			// request like Amazon S3.
-			if parseRangeErr == errInvalidRange {
-				writeErrorResponse(w, ErrInvalidRange, r.URL)
-				return
-			}
-
-			// log the error.
-			logger.LogIf(ctx, parseRangeErr)
-		}
-	}
-
 	var lock = noLock
 	if !cpSrcDstSame {
 		lock = readLock
 	}
 
+	var rs *HTTPRangeSpec
 	gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, lock, srcOpts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -735,9 +752,27 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	var reader io.Reader
 	var length = srcInfo.Size
+
+	// Set the actual size to the decrypted size if encrypted.
+	actualSize := srcInfo.Size
+	if crypto.IsEncrypted(srcInfo.UserDefined) {
+		actualSize, err = srcInfo.DecryptedSize()
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+	}
+
 	// No need to compress for remote etcd calls
 	// Pass the decompressed stream to such calls.
-	if srcInfo.IsCompressed() && !isRemoteCallRequired(ctx, srcBucket, dstBucket, objectAPI) {
+	isCompressed := objectAPI.IsCompressionSupported() && isCompressible(r.Header, srcObject) && !isRemoteCallRequired(ctx, srcBucket, dstBucket, objectAPI)
+	if isCompressed {
+		// Storing the compression metadata.
+		srcInfo.UserDefined[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
+		srcInfo.UserDefined[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
+		// Remove all source encrypted related metadata to
+		// avoid copying them in target object.
+		crypto.RemoveInternalEntries(srcInfo.UserDefined)
 		// Open a pipe for compression.
 		// Where pipeWriter is piped to srcInfo.Reader.
 		// gr writes to pipeWriter.
@@ -760,45 +795,58 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		reader = gr
 	}
 
-	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", srcInfo.Size)
+	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualSize)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// Save the original size for later use when we want to copy
-	// encrypted file into an unencrypted one.
-	size := srcInfo.Size
-
 	var encMetadata = make(map[string]string)
-	if objectAPI.IsEncryptionSupported() && !srcInfo.IsCompressed() {
+	if objectAPI.IsEncryptionSupported() && !isCompressed {
+		// Encryption parameters not applicable for this object.
+		if !crypto.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header) {
+			writeErrorResponse(w, toAPIErrorCode(errInvalidEncryptionParameters), r.URL)
+			return
+		}
+		// Encryption parameters not present for this object.
+		if crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && !crypto.SSECopy.IsRequested(r.Header) {
+			writeErrorResponse(w, ErrInvalidSSECustomerAlgorithm, r.URL)
+			return
+		}
 		var oldKey, newKey []byte
 		sseCopyS3 := crypto.S3.IsEncrypted(srcInfo.UserDefined)
-		sseCopyC := crypto.SSECopy.IsRequested(r.Header)
+		sseCopyC := crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header)
 		sseC := crypto.SSEC.IsRequested(r.Header)
 		sseS3 := crypto.S3.IsRequested(r.Header)
-		if sseC || sseS3 {
-			if sseC {
-				newKey, err = ParseSSECustomerRequest(r)
-			}
+
+		isSourceEncrypted := sseCopyC || sseCopyS3
+		isTargetEncrypted := sseC || sseS3
+
+		if sseC {
+			newKey, err = ParseSSECustomerRequest(r)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
 		}
-		// AWS S3 implementation requires us to only rotate keys
-		// when/ both keys are provided and destination is same
-		// otherwise we proceed to encrypt/decrypt.
-		if sseCopyC && sseC && cpSrcDstSame {
-			// Get the old key which needs to be rotated.
-			oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined)
-			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-				return
+
+		// If src == dst and either
+		// - the object is encrypted using SSE-C and two different SSE-C keys are present
+		// - the object is encrypted using SSE-S3 and the SSE-S3 header is present
+		// than execute a key rotation.
+		if cpSrcDstSame && ((sseCopyC && sseC) || (sseS3 && sseCopyS3)) {
+			if sseCopyC && sseC {
+				oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined)
+				if err != nil {
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
 			}
 			for k, v := range srcInfo.UserDefined {
 				encMetadata[k] = v
 			}
+
+			// In case of SSE-S3 oldKey and newKey aren't used - the KMS manages the keys.
 			if err = rotateKey(oldKey, newKey, srcBucket, srcObject, encMetadata); err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -807,42 +855,43 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			// Since we are rotating the keys, make sure to update the metadata.
 			srcInfo.metadataOnly = true
 		} else {
-			if sseCopyC || sseCopyS3 {
+			if isSourceEncrypted || isTargetEncrypted {
 				// We are not only copying just metadata instead
 				// we are creating a new object at this point, even
 				// if source and destination are same objects.
 				srcInfo.metadataOnly = false
-				if sseC || sseS3 {
-					size = srcInfo.Size
-				}
 			}
-			if sseC || sseS3 {
+
+			// Calculate the size of the target object
+			var targetSize int64
+
+			switch {
+			case !isSourceEncrypted && !isTargetEncrypted:
+				fallthrough
+			case isSourceEncrypted && isTargetEncrypted:
+				targetSize = srcInfo.Size
+			// Source not encrypted and target encrypted
+			case !isSourceEncrypted && isTargetEncrypted:
+				targetSize = srcInfo.EncryptedSize()
+			case isSourceEncrypted && !isTargetEncrypted:
+				targetSize, _ = srcInfo.DecryptedSize()
+			}
+
+			if isTargetEncrypted {
 				reader, err = newEncryptReader(reader, newKey, dstBucket, dstObject, encMetadata, sseS3)
 				if err != nil {
 					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 					return
 				}
-				// We are not only copying just metadata instead
-				// we are creating a new object at this point, even
-				// if source and destination are same objects.
-				srcInfo.metadataOnly = false
-				if !sseCopyC && !sseCopyS3 {
-					size = srcInfo.EncryptedSize()
-				}
-			} else {
-				if sseCopyC || sseCopyS3 {
-					size, _ = srcInfo.DecryptedSize()
-					delete(srcInfo.UserDefined, crypto.SSEIV)
-					delete(srcInfo.UserDefined, crypto.SSESealAlgorithm)
-					delete(srcInfo.UserDefined, crypto.SSECSealedKey)
-					delete(srcInfo.UserDefined, crypto.SSEMultipart)
-					delete(srcInfo.UserDefined, crypto.S3SealedKey)
-					delete(srcInfo.UserDefined, crypto.S3KMSSealedKey)
-					delete(srcInfo.UserDefined, crypto.S3KMSKeyID)
-				}
 			}
 
-			srcInfo.Reader, err = hash.NewReader(reader, size, "", "", size) // do not try to verify encrypted content
+			if isSourceEncrypted {
+				// Remove all source encrypted related metadata to
+				// avoid copying them in target object.
+				crypto.RemoveInternalEntries(srcInfo.UserDefined)
+			}
+
+			srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", targetSize) // do not try to verify encrypted content
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -870,7 +919,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	// metadataOnly is true indicating that we are not overwriting the object.
 	// if encryption is enabled we do not need explicit "REPLACE" metadata to
 	// be enabled as well - this is to allow for key-rotation.
-	if !isMetadataReplace(r.Header) && srcInfo.metadataOnly && !crypto.SSEC.IsEncrypted(srcInfo.UserDefined) {
+	if !isMetadataReplace(r.Header) && srcInfo.metadataOnly && !crypto.IsEncrypted(srcInfo.UserDefined) {
 		// If x-amz-metadata-directive is not set to REPLACE then we need
 		// to error out if source and destination are same.
 		writeErrorResponse(w, ErrInvalidCopyDest, r.URL)
@@ -934,19 +983,20 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		host, port = "", ""
 	}
 
-	if srcInfo.IsCompressed() {
-		objInfo.Size = srcInfo.GetActualSize()
+	if objInfo.IsCompressed() {
+		objInfo.Size = actualSize
 	}
 
 	// Notify object created event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedCopy,
-		BucketName: dstBucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedCopy,
+		BucketName:   dstBucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 }
 
@@ -961,12 +1011,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObject")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -1057,22 +1109,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		putObject = objectAPI.PutObject
 	)
 	reader = r.Body
-	switch rAuthType {
-	default:
-		// For all unknown auth types return error.
-		writeErrorResponse(w, ErrAccessDenied, r.URL)
+
+	// Check if put is allowed
+	if s3Err = isPutAllowed(rAuthType, bucket, object, r); s3Err != ErrNone {
+		writeErrorResponse(w, s3Err, r.URL)
 		return
-	case authTypeAnonymous:
-		if !globalPolicySys.IsAllowed(policy.Args{
-			Action:          policy.PutObjectAction,
-			BucketName:      bucket,
-			ConditionValues: getConditionValues(r, ""),
-			IsOwner:         false,
-			ObjectName:      object,
-		}) {
-			writeErrorResponse(w, ErrAccessDenied, r.URL)
-			return
-		}
+	}
+
+	switch rAuthType {
 	case authTypeStreamingSigned:
 		// Initialize stream signature verifier.
 		reader, s3Err = newSignV4ChunkedReader(r)
@@ -1119,7 +1163,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			_, cerr := io.CopyN(snappyWriter, actualReader, actualSize)
 			snappyWriter.Close()
 			pipeWriter.CloseWithError(cerr)
-
 		}()
 
 		// Set compression metrics.
@@ -1201,13 +1244,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	// Notify object created event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedPut,
-		BucketName: bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedPut,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 }
 
@@ -1222,12 +1266,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "NewMultipartUpload")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -1314,12 +1360,14 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObjectPart")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -1347,6 +1395,11 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	uploadID := r.URL.Query().Get("uploadId")
 	partIDString := r.URL.Query().Get("partNumber")
 
@@ -1400,11 +1453,17 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	defer gr.Close()
 	srcInfo := gr.ObjInfo
 
-	var actualPartSize int64
-	actualPartSize = srcInfo.Size
+	actualPartSize := srcInfo.Size
+	if crypto.IsEncrypted(srcInfo.UserDefined) {
+		actualPartSize, err = srcInfo.DecryptedSize()
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+	}
 
 	// Special care for CopyObjectPart
-	if partRangeErr := checkCopyPartRangeWithSize(rs, srcInfo.Size); partRangeErr != nil {
+	if partRangeErr := checkCopyPartRangeWithSize(rs, actualPartSize); partRangeErr != nil {
 		writeCopyPartErr(w, partRangeErr, r.URL)
 		return
 	}
@@ -1415,21 +1474,10 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	}
 
 	// Get the object offset & length
-	startOffset, length, _ := rs.GetOffsetLength(srcInfo.Size)
-
-	if rangeHeader != "" {
-		actualPartSize = length
-	}
-
-	if objectAPI.IsEncryptionSupported() {
-		if crypto.IsEncrypted(srcInfo.UserDefined) {
-			decryptedSize, decryptErr := srcInfo.DecryptedSize()
-			if decryptErr != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-				return
-			}
-			startOffset, length, _ = rs.GetOffsetLength(decryptedSize)
-		}
+	startOffset, length, err := rs.GetOffsetLength(actualPartSize)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
 	}
 
 	/// maximum copy size for multipart objects in a single operation
@@ -1438,11 +1486,21 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
+	actualPartSize = length
 	var reader io.Reader
-	var getLength = length
 
-	// Need to decompress only for range-enabled copy parts.
-	if srcInfo.IsCompressed() && rangeHeader != "" {
+	var li ListPartsInfo
+	li, err = objectAPI.ListObjectParts(ctx, dstBucket, dstObject, uploadID, 0, 1)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Read compression metadata preserved in the init multipart for the decision.
+	_, compressPart := li.UserDefined[ReservedMetadataPrefix+"compression"]
+	isCompressed := compressPart
+	// Compress only if the compression is enabled during initial multipart.
+	if isCompressed {
 		// Open a pipe for compression.
 		// Where pipeWriter is piped to srcInfo.Reader.
 		// gr writes to pipeWriter.
@@ -1468,15 +1526,13 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	if objectAPI.IsEncryptionSupported() && !srcInfo.IsCompressed() {
-		var li ListPartsInfo
-		li, err = objectAPI.ListObjectParts(ctx, dstBucket, dstObject, uploadID, 0, 1)
-		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-			return
-		}
+	if objectAPI.IsEncryptionSupported() && !isCompressed {
 		if crypto.IsEncrypted(li.UserDefined) {
-			if !hasServerSideEncryptionHeader(r.Header) {
+			if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(li.UserDefined) {
+				writeErrorResponse(w, ErrSSEMultipartEncrypted, r.URL)
+				return
+			}
+			if crypto.S3.IsEncrypted(li.UserDefined) && crypto.SSEC.IsRequested(r.Header) {
 				writeErrorResponse(w, ErrSSEMultipartEncrypted, r.URL)
 				return
 			}
@@ -1508,18 +1564,18 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			}
 
 			info := ObjectInfo{Size: length}
-			size := info.EncryptedSize()
-			srcInfo.Reader, err = hash.NewReader(reader, size, "", "", actualPartSize)
+			srcInfo.Reader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", length)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
 		}
 	}
+
 	// Copy source object to destination, if source and destination
 	// object is same then only metadata is updated.
-	partInfo, err := objectAPI.CopyObjectPart(ctx, srcBucket, srcObject, dstBucket,
-		dstObject, uploadID, partID, startOffset, getLength, srcInfo, srcOpts, dstOpts)
+	partInfo, err := objectAPI.CopyObjectPart(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID,
+		startOffset, length, srcInfo, srcOpts, dstOpts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -1536,12 +1592,14 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectPart")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -1611,41 +1669,30 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		md5hex    = hex.EncodeToString(md5Bytes)
 		sha256hex = ""
 		reader    io.Reader
+		s3Error   APIErrorCode
 	)
 	reader = r.Body
 
-	switch rAuthType {
-	default:
-		// For all unknown auth types return error.
-		writeErrorResponse(w, ErrAccessDenied, r.URL)
+	if s3Error = isPutAllowed(rAuthType, bucket, object, r); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
 		return
-	case authTypeAnonymous:
-		if !globalPolicySys.IsAllowed(policy.Args{
-			Action:          policy.PutObjectAction,
-			BucketName:      bucket,
-			ConditionValues: getConditionValues(r, ""),
-			IsOwner:         false,
-			ObjectName:      object,
-		}) {
-			writeErrorResponse(w, ErrAccessDenied, r.URL)
-			return
-		}
+	}
+
+	switch rAuthType {
 	case authTypeStreamingSigned:
 		// Initialize stream signature verifier.
-		var s3Error APIErrorCode
 		reader, s3Error = newSignV4ChunkedReader(r)
 		if s3Error != ErrNone {
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	case authTypeSignedV2, authTypePresignedV2:
-		s3Error := isReqAuthenticatedV2(r)
-		if s3Error != ErrNone {
+		if s3Error = isReqAuthenticatedV2(r); s3Error != ErrNone {
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error := reqSignatureV4Verify(r, globalServerConfig.GetRegion()); s3Error != ErrNone {
+		if s3Error = reqSignatureV4Verify(r, globalServerConfig.GetRegion()); s3Error != ErrNone {
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
@@ -1710,6 +1757,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		}
 	}
 
+	isEncrypted := false
 	if objectAPI.IsEncryptionSupported() && !isCompressed {
 		var li ListPartsInfo
 		li, err = objectAPI.ListObjectParts(ctx, bucket, object, uploadID, 0, 1)
@@ -1718,7 +1766,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			return
 		}
 		if crypto.IsEncrypted(li.UserDefined) {
-			if !hasServerSideEncryptionHeader(r.Header) {
+			isEncrypted = true
+			if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(li.UserDefined) {
 				writeErrorResponse(w, ErrSSEMultipartEncrypted, r.URL)
 				return
 			}
@@ -1746,7 +1795,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			mac.Write(partIDbin[:])
 			partEncryptionKey := mac.Sum(nil)
 
-			reader, err = sio.EncryptReader(reader, sio.Config{Key: partEncryptionKey})
+			reader, err = sio.EncryptReader(hashReader, sio.Config{Key: partEncryptionKey})
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -1762,7 +1811,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	}
 
 	putObjectPart := objectAPI.PutObjectPart
-	if api.CacheAPI() != nil && !hasServerSideEncryptionHeader(r.Header) {
+	if api.CacheAPI() != nil && !isEncrypted {
 		putObjectPart = api.CacheAPI().PutObjectPart
 	}
 	partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, hashReader, opts)
@@ -1787,6 +1836,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AbortMultipartUpload")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -1814,11 +1865,16 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 		}
 	}
 
-	uploadID, _, _, _ := getObjectResources(r.URL.Query())
+	uploadID, _, _, _, s3Error := getObjectResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 	if err := abortMultipartUpload(ctx, bucket, object, uploadID); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
+
 	writeSuccessNoContent(w)
 }
 
@@ -1826,6 +1882,8 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectParts")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -1841,7 +1899,11 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
+	uploadID, partNumberMarker, maxParts, _, s3Error := getObjectResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 	if partNumberMarker < 0 {
 		writeErrorResponse(w, ErrInvalidPartNumberMarker, r.URL)
 		return
@@ -1866,6 +1928,8 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CompleteMultipartUpload")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -1890,7 +1954,11 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	// Get upload id.
-	uploadID, _, _, _ := getObjectResources(r.URL.Query())
+	uploadID, _, _, _, s3Error := getObjectResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 
 	completeMultipartBytes, err := goioutil.ReadAll(r.Body)
 	if err != nil {
@@ -1964,13 +2032,14 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	// Notify object created event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedCompleteMultipartUpload,
-		BucketName: bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedCompleteMultipartUpload,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 }
 
@@ -1980,6 +2049,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteObject")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -2013,21 +2084,18 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			}
 			return
 		}
-	} else {
-		getBucketInfo := objectAPI.GetBucketInfo
-		if api.CacheAPI() != nil {
-			getBucketInfo = api.CacheAPI().GetBucketInfo
-		}
-		if _, err := getBucketInfo(ctx, bucket); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-			return
-		}
 	}
 
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
-	// Ignore delete object errors while replying to client, since we are
-	// suppposed to reply only 204. Additionally log the error for
-	// investigation.
-	deleteObject(ctx, objectAPI, api.CacheAPI(), bucket, object, r)
+	if err := deleteObject(ctx, objectAPI, api.CacheAPI(), bucket, object, r); err != nil {
+		switch toAPIErrorCode(err) {
+		case ErrNoSuchBucket:
+			// When bucket doesn't exist specially handle it.
+			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+			return
+		}
+		logger.LogIf(ctx, err)
+		// Ignore delete object errors while replying to client, since we are suppposed to reply only 204.
+	}
 	writeSuccessNoContent(w)
 }

cmd/object-handlers_test.go

@@ -2163,23 +2163,6 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		}
 	}
 
-	// Test for Anonymous/unsigned http request.
-	newCopyAnonObject := "new-anon-obj"
-	anonReq, err := newTestRequest("PUT", getCopyObjectURL("", bucketName, newCopyAnonObject), 0, nil)
-	if err != nil {
-		t.Fatalf("Minio %s: Failed to create an anonymous request for %s/%s: <ERROR> %v",
-			instanceType, bucketName, "new-anon-obj", err)
-	}
-
-	// Below is how CopyObjectHandler is registered.
-	// bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?")
-	// Its necessary to set the "X-Amz-Copy-Source" header for the request to be accepted by the handler.
-	anonReq.Header.Set("X-Amz-Copy-Source", url.QueryEscape("/"+bucketName+"/"+anonObject))
-	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
-	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
-	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPICopyObjectHandler", bucketName, newCopyAnonObject, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, newCopyAnonObject))
-
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
 	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
@@ -2378,14 +2361,16 @@ func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketNam
 			req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, objectName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 
 			if err != nil {
-				t.Fatalf("Failed to create HTTP request for NewMultipart request: <ERROR> %v", err)
+				t.Errorf("Failed to create HTTP request for NewMultipart request: <ERROR> %v", err)
+				return
 			}
 			// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
 			// Call the ServeHTTP to executes the registered handler.
 			apiRouter.ServeHTTP(rec, req)
 			// Assert the response code with the expected status.
 			if rec.Code != http.StatusOK {
-				t.Fatalf("Minio %s:  Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusOK, rec.Code)
+				t.Errorf("Minio %s:  Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusOK, rec.Code)
+				return
 			}
 			// decode the response body.
 			decoder := xml.NewDecoder(rec.Body)
@@ -2393,7 +2378,8 @@ func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketNam
 
 			err = decoder.Decode(multipartResponse)
 			if err != nil {
-				t.Fatalf("Minio %s: Error decoding the recorded response Body", instanceType)
+				t.Errorf("Minio %s: Error decoding the recorded response Body", instanceType)
+				return
 			}
 			// push the obtained upload ID from the response into the array.
 			testUploads.Lock()

cmd/policy.go

@@ -17,10 +17,12 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"net/http"
 	"path"
+	"strings"
 	"sync"
 	"time"
 
@@ -115,7 +117,7 @@ func (sys *PolicySys) refresh(objAPI ObjectLayer) error {
 			logger.Info("Found in-consistent bucket policies, Migrating them for Bucket: (%s)", bucket.Name)
 			config.Version = policy.DefaultVersion
 
-			if err = savePolicyConfig(objAPI, bucket.Name, config); err != nil {
+			if err = savePolicyConfig(context.Background(), objAPI, bucket.Name, config); err != nil {
 				logger.LogIf(context.Background(), err)
 				return err
 			}
@@ -131,25 +133,46 @@ func (sys *PolicySys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	// Load PolicySys once during boot.
-	if err := sys.refresh(objAPI); err != nil {
-		return err
-	}
-
-	// Refresh PolicySys in background.
-	go func() {
-		ticker := time.NewTicker(globalRefreshBucketPolicyInterval)
-		defer ticker.Stop()
-		for {
-			select {
-			case <-globalServiceDoneCh:
-				return
-			case <-ticker.C:
-				sys.refresh(objAPI)
+	defer func() {
+		// Refresh PolicySys in background.
+		go func() {
+			ticker := time.NewTicker(globalRefreshBucketPolicyInterval)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-globalServiceDoneCh:
+					return
+				case <-ticker.C:
+					sys.refresh(objAPI)
+				}
 			}
-		}
+		}()
 	}()
-	return nil
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing policy needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			// Load PolicySys once during boot.
+			if err := sys.refresh(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for policy subsystem to be initialized..")
+					continue
+				}
+				return err
+			}
+			return nil
+		}
+	}
 }
 
 // NewPolicySys - creates new policy system.
@@ -192,7 +215,7 @@ func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, err
 	// Construct path to policy.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
 
-	reader, err := readConfig(context.Background(), objAPI, configFile)
+	configData, err := readConfig(context.Background(), objAPI, configFile)
 	if err != nil {
 		if err == errConfigNotFound {
 			err = BucketPolicyNotFound{Bucket: bucketName}
@@ -201,10 +224,10 @@ func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, err
 		return nil, err
 	}
 
-	return policy.ParseConfig(reader, bucketName)
+	return policy.ParseConfig(bytes.NewReader(configData), bucketName)
 }
 
-func savePolicyConfig(objAPI ObjectLayer, bucketName string, bucketPolicy *policy.Policy) error {
+func savePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, bucketPolicy *policy.Policy) error {
 	data, err := json.Marshal(bucketPolicy)
 	if err != nil {
 		return err
@@ -213,7 +236,7 @@ func savePolicyConfig(objAPI ObjectLayer, bucketName string, bucketPolicy *polic
 	// Construct path to policy.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
 
-	return saveConfig(objAPI, configFile, data)
+	return saveConfig(ctx, objAPI, configFile, data)
 }
 
 func removePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {

cmd/routers.go

@@ -82,6 +82,8 @@ var globalHandlers = []HandlerFunc{
 	// routes them accordingly. Client receives a HTTP error for
 	// invalid/unsupported signatures.
 	setAuthHandler,
+	// Enforce rules specific for TLS requests
+	setSSETLSHandler,
 	// filters HTTP headers which are treated as metadata and are reserved
 	// for internal use only.
 	filterReservedMetadata,
@@ -99,6 +101,9 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
 		registerDistXLRouters(router, endpoints)
 	}
 
+	// Add STS router only enabled if etcd is configured.
+	registerSTSRouter(router)
+
 	// Add Admin RPC router
 	registerAdminRPCRouter(router)
 

cmd/server-main.go

@@ -221,17 +221,21 @@ func serverMain(ctx *cli.Context) {
 	// Handle all server command args.
 	serverHandleCmdArgs(ctx)
 
-	// Handle all server environment vars.
-	serverHandleEnvVars()
-
 	// Create certs path.
 	logger.FatalIf(createConfigDir(), "Unable to initialize configuration files")
 
-	// Check and load SSL certificates.
+	// Check and load TLS certificates.
 	var err error
-	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
+	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
 	logger.FatalIf(err, "Unable to load the TLS configuration")
 
+	// Check and load Root CAs.
+	globalRootCAs, err = getRootCAs(getCADir())
+	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
+
+	// Handle all server environment vars.
+	serverHandleEnvVars()
+
 	// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
 	if globalIsDistXL {
 		if globalEndpoints.IsHTTPS() && !globalIsSSL {
@@ -358,11 +362,17 @@ func serverMain(ctx *cli.Context) {
 		logger.FatalIf(err, "Unable to initialize disk caching")
 	}
 
+	// Create new IAM system.
+	globalIAMSys = NewIAMSys()
+	if err = globalIAMSys.Init(newObject); err != nil {
+		logger.Fatal(err, "Unable to initialize IAM system")
+	}
+
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
 
 	// Initialize policy system.
-	if err := globalPolicySys.Init(newObject); err != nil {
+	if err = globalPolicySys.Init(newObject); err != nil {
 		logger.Fatal(err, "Unable to initialize policy system")
 	}
 
@@ -370,7 +380,7 @@ func serverMain(ctx *cli.Context) {
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 
 	// Initialize notification system.
-	if err := globalNotificationSys.Init(newObject); err != nil {
+	if err = globalNotificationSys.Init(newObject); err != nil {
 		logger.Fatal(err, "Unable to initialize notification system")
 	}
 

cmd/server-startup-msg.go

@@ -116,10 +116,12 @@ func printServerCommonMsg(apiEndpoints []string) {
 
 	// Colorize the message and print.
 	logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
-	logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
-	logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
-	if region != "" {
-		logger.StartupMessage(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
+	if isTerminal() {
+		logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+		if region != "" {
+			logger.StartupMessage(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
+		}
 	}
 	printEventNotifiers()
 
@@ -151,13 +153,15 @@ func printCLIAccessMsg(endPoint string, alias string) {
 	cred := globalServerConfig.GetCredential()
 
 	// Configure 'mc', following block prints platform specific information for minio client.
-	logger.StartupMessage(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
-	if runtime.GOOS == globalWindowsOSName {
-		mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
-		logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-	} else {
-		mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
-		logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+	if isTerminal() {
+		logger.StartupMessage(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
+		if runtime.GOOS == globalWindowsOSName {
+			mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
+			logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+		} else {
+			mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
+			logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+		}
 	}
 }
 

cmd/server_test.go

@@ -1046,7 +1046,8 @@ func (s *TestSuiteCommon) TestPutBucket(c *check) {
 			client := http.Client{Transport: s.transport}
 			response, err := client.Do(request)
 			if err != nil {
-				c.Fatalf("Put bucket Failed: <ERROR> %s", err)
+				c.Errorf("Put bucket Failed: <ERROR> %s", err)
+				return
 			}
 			defer response.Body.Close()
 		}()

cmd/signature-v2.go

@@ -27,6 +27,8 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Signature and API related constants.
@@ -68,7 +70,14 @@ func doesPolicySignatureV2Match(formValues http.Header) APIErrorCode {
 	cred := globalServerConfig.GetCredential()
 	accessKey := formValues.Get("AWSAccessKeyId")
 	if accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 	policy := formValues.Get("Policy")
 	signature := formValues.Get("Signature")
@@ -146,7 +155,14 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 
 	// Validate if access key id same.
 	if accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 
 	// Make sure the request has not expired.
@@ -165,7 +181,7 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 		return ErrInvalidRequest
 	}
 
-	expectedSignature := preSignatureV2(r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires)
+	expectedSignature := preSignatureV2(cred, r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires)
 	if !compareSignatureV2(gotSignature, expectedSignature) {
 		return ErrSignatureDoesNotMatch
 	}
@@ -173,6 +189,29 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 	return ErrNone
 }
 
+func getReqAccessKeyV2(r *http.Request) (string, bool, APIErrorCode) {
+	if accessKey := r.URL.Query().Get("AWSAccessKeyId"); accessKey != "" {
+		owner, s3Err := checkKeyValid(accessKey)
+		return accessKey, owner, s3Err
+	}
+
+	// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
+	// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
+	authFields := strings.Split(r.Header.Get("Authorization"), " ")
+	if len(authFields) != 2 {
+		return "", false, ErrMissingFields
+	}
+
+	// Then will be splitting on ":", this will seprate `AWSAccessKeyId` and `Signature` string.
+	keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":")
+	if len(keySignFields) != 2 {
+		return "", false, ErrMissingFields
+	}
+
+	owner, s3Err := checkKeyValid(keySignFields[0])
+	return keySignFields[0], owner, s3Err
+}
+
 // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
 // Signature = Base64( HMAC-SHA1( YourSecretKey, UTF-8-Encoding-Of( StringToSign ) ) );
 //
@@ -193,41 +232,43 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 //     - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
 // returns true if matches, false otherwise. if error is not nil then it is always false
 
-func validateV2AuthHeader(v2Auth string) APIErrorCode {
+func validateV2AuthHeader(r *http.Request) (auth.Credentials, APIErrorCode) {
+	var cred auth.Credentials
+	v2Auth := r.Header.Get("Authorization")
 	if v2Auth == "" {
-		return ErrAuthHeaderEmpty
+		return cred, ErrAuthHeaderEmpty
 	}
+
 	// Verify if the header algorithm is supported or not.
 	if !strings.HasPrefix(v2Auth, signV2Algorithm) {
-		return ErrSignatureVersionNotSupported
+		return cred, ErrSignatureVersionNotSupported
 	}
 
-	// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
-	// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
-	authFields := strings.Split(v2Auth, " ")
-	if len(authFields) != 2 {
-		return ErrMissingFields
-	}
-
-	// Then will be splitting on ":", this will seprate `AWSAccessKeyId` and `Signature` string.
-	keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":")
-	if len(keySignFields) != 2 {
-		return ErrMissingFields
+	accessKey, owner, apiErr := getReqAccessKeyV2(r)
+	if apiErr != ErrNone {
+		return cred, apiErr
 	}
 
+	cred = globalServerConfig.GetCredential()
 	// Access credentials.
-	cred := globalServerConfig.GetCredential()
-	if keySignFields[0] != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+	if !owner {
+		if globalIAMSys == nil {
+			return cred, ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return cred, ErrInvalidAccessKeyID
+		}
 	}
 
-	return ErrNone
+	return cred, ErrNone
 }
 
 func doesSignV2Match(r *http.Request) APIErrorCode {
 	v2Auth := r.Header.Get("Authorization")
-
-	if apiError := validateV2AuthHeader(v2Auth); apiError != ErrNone {
+	cred, apiError := validateV2AuthHeader(r)
+	if apiError != ErrNone {
 		return apiError
 	}
 
@@ -249,12 +290,12 @@ func doesSignV2Match(r *http.Request) APIErrorCode {
 		return ErrInvalidRequest
 	}
 
-	prefix := fmt.Sprintf("%s %s:", signV2Algorithm, globalServerConfig.GetCredential().AccessKey)
+	prefix := fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKey)
 	if !strings.HasPrefix(v2Auth, prefix) {
 		return ErrSignatureDoesNotMatch
 	}
 	v2Auth = v2Auth[len(prefix):]
-	expectedAuth := signatureV2(r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header)
+	expectedAuth := signatureV2(cred, r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header)
 	if !compareSignatureV2(v2Auth, expectedAuth) {
 		return ErrSignatureDoesNotMatch
 	}
@@ -268,15 +309,13 @@ func calculateSignatureV2(stringToSign string, secret string) string {
 }
 
 // Return signature-v2 for the presigned request.
-func preSignatureV2(method string, encodedResource string, encodedQuery string, headers http.Header, expires string) string {
-	cred := globalServerConfig.GetCredential()
+func preSignatureV2(cred auth.Credentials, method string, encodedResource string, encodedQuery string, headers http.Header, expires string) string {
 	stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, expires)
 	return calculateSignatureV2(stringToSign, cred.SecretKey)
 }
 
 // Return the signature v2 of a given request.
-func signatureV2(method string, encodedResource string, encodedQuery string, headers http.Header) string {
-	cred := globalServerConfig.GetCredential()
+func signatureV2(cred auth.Credentials, method string, encodedResource string, encodedQuery string, headers http.Header) string {
 	stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, "")
 	signature := calculateSignatureV2(stringToSign, cred.SecretKey)
 	return signature

cmd/signature-v2_test.go

@@ -223,7 +223,12 @@ func TestValidateV2AuthHeader(t *testing.T) {
 	for i, testCase := range testCases {
 		t.Run(fmt.Sprintf("Case %d AuthStr \"%s\".", i+1, testCase.authString), func(t *testing.T) {
 
-			actualErrCode := validateV2AuthHeader(testCase.authString)
+			req := &http.Request{
+				Header: make(http.Header),
+				URL:    &url.URL{},
+			}
+			req.Header.Set("Authorization", testCase.authString)
+			_, actualErrCode := validateV2AuthHeader(req)
 
 			if testCase.expectedError != actualErrCode {
 				t.Errorf("Expected the error code to be %v, got %v.", testCase.expectedError, actualErrCode)

cmd/signature-v4-parser.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"net/http"
 	"net/url"
 	"strings"
 	"time"
@@ -46,6 +47,24 @@ func (c credentialHeader) getScope() string {
 	}, "/")
 }
 
+func getReqAccessKeyV4(r *http.Request, region string) (string, bool, APIErrorCode) {
+	ch, err := parseCredentialHeader("Credential="+r.URL.Query().Get("X-Amz-Credential"), region)
+	if err != ErrNone {
+		// Strip off the Algorithm prefix.
+		v4Auth := strings.TrimPrefix(r.Header.Get("Authorization"), signV4Algorithm)
+		authFields := strings.Split(strings.TrimSpace(v4Auth), ",")
+		if len(authFields) != 3 {
+			return "", false, ErrMissingFields
+		}
+		ch, err = parseCredentialHeader(authFields[0], region)
+		if err != ErrNone {
+			return "", false, err
+		}
+	}
+	owner, s3Err := checkKeyValid(ch.accessKey)
+	return ch.accessKey, owner, s3Err
+}
+
 // parse credentialHeader string into its structured form.
 func parseCredentialHeader(credElement string, region string) (ch credentialHeader, aec APIErrorCode) {
 	creds := strings.Split(strings.TrimSpace(credElement), "=")

cmd/signature-v4-utils.go

@@ -100,6 +100,23 @@ func isValidRegion(reqRegion string, confRegion string) bool {
 	return reqRegion == confRegion
 }
 
+// check if the access key is valid and recognized, additionally
+// also returns if the access key is owner/admin.
+func checkKeyValid(accessKey string) (bool, APIErrorCode) {
+	var owner = true
+	if globalServerConfig.GetCredential().AccessKey != accessKey {
+		if globalIAMSys == nil {
+			return false, ErrInvalidAccessKeyID
+		}
+		// Check if the access key is part of users credentials.
+		if _, ok := globalIAMSys.GetUser(accessKey); !ok {
+			return false, ErrInvalidAccessKeyID
+		}
+		owner = false
+	}
+	return owner, ErrNone
+}
+
 // sumHMAC calculate hmac between two input byte array.
 func sumHMAC(key []byte, data []byte) []byte {
 	hash := hmac.New(sha256.New, key)

cmd/signature-v4.go

@@ -175,7 +175,14 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
 
 	// Verify if the access key id matches.
 	if credHeader.accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(credHeader.accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 
 	// Get signing key.
@@ -211,7 +218,14 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 
 	// Verify if the access key id matches.
 	if pSignValues.Credential.accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(pSignValues.Credential.accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 
 	// Extract all the signed headers along with its values.
@@ -335,7 +349,14 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string) AP
 
 	// Verify if the access key id matches.
 	if signV4Values.Credential.accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(signV4Values.Credential.accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 
 	// Extract date, if not present throw error.

cmd/streaming-signature-v4.go

@@ -64,7 +64,7 @@ func getChunkSignature(cred auth.Credentials, seedSignature string, region strin
 //     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
 // returns signature, error otherwise if the signature mismatches or any other
 // error while parsing and validating.
-func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature string, region string, date time.Time, errCode APIErrorCode) {
+func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature string, region string, date time.Time, errCode APIErrorCode) {
 	// Copy request.
 	req := *r
 
@@ -74,7 +74,7 @@ func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature s
 	// Parse signature version '4' header.
 	signV4Values, errCode := parseSignV4(v4Auth, globalServerConfig.GetRegion())
 	if errCode != ErrNone {
-		return "", "", time.Time{}, errCode
+		return cred, "", "", time.Time{}, errCode
 	}
 
 	// Payload streaming.
@@ -82,17 +82,26 @@ func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature s
 
 	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
 	if payload != req.Header.Get("X-Amz-Content-Sha256") {
-		return "", "", time.Time{}, ErrContentSHA256Mismatch
+		return cred, "", "", time.Time{}, ErrContentSHA256Mismatch
 	}
 
 	// Extract all the signed headers along with its values.
 	extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
 	if errCode != ErrNone {
-		return "", "", time.Time{}, errCode
+		return cred, "", "", time.Time{}, errCode
 	}
+
+	cred = globalServerConfig.GetCredential()
 	// Verify if the access key id matches.
 	if signV4Values.Credential.accessKey != cred.AccessKey {
-		return "", "", time.Time{}, ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return cred, "", "", time.Time{}, ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(signV4Values.Credential.accessKey)
+		if !ok {
+			return cred, "", "", time.Time{}, ErrInvalidAccessKeyID
+		}
 	}
 
 	// Verify if region is valid.
@@ -102,14 +111,14 @@ func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature s
 	var dateStr string
 	if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
 		if dateStr = r.Header.Get("Date"); dateStr == "" {
-			return "", "", time.Time{}, ErrMissingDateHeader
+			return cred, "", "", time.Time{}, ErrMissingDateHeader
 		}
 	}
 	// Parse date header.
 	var err error
 	date, err = time.Parse(iso8601Format, dateStr)
 	if err != nil {
-		return "", "", time.Time{}, ErrMalformedDate
+		return cred, "", "", time.Time{}, ErrMalformedDate
 	}
 
 	// Query string.
@@ -129,11 +138,11 @@ func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature s
 
 	// Verify if signature match.
 	if !compareSignatureV4(newSignature, signV4Values.Signature) {
-		return "", "", time.Time{}, ErrSignatureDoesNotMatch
+		return cred, "", "", time.Time{}, ErrSignatureDoesNotMatch
 	}
 
 	// Return caculated signature.
-	return newSignature, region, date, ErrNone
+	return cred, newSignature, region, date, ErrNone
 }
 
 const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB
@@ -151,10 +160,7 @@ var errMalformedEncoding = errors.New("malformed chunked encoding")
 // NewChunkedReader is not needed by normal applications. The http package
 // automatically decodes chunking when reading response bodies.
 func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, APIErrorCode) {
-	// Access credentials.
-	cred := globalServerConfig.GetCredential()
-
-	seedSignature, region, seedDate, errCode := calculateSeedSignature(cred, req)
+	cred, seedSignature, region, seedDate, errCode := calculateSeedSignature(req)
 	if errCode != ErrNone {
 		return nil, errCode
 	}

cmd/sts-errors.go

@@ -0,0 +1,119 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+)
+
+// writeSTSErrorRespone writes error headers
+func writeSTSErrorResponse(w http.ResponseWriter, errorCode STSErrorCode) {
+	stsError := getSTSError(errorCode)
+	// Generate error response.
+	stsErrorResponse := getSTSErrorResponse(stsError)
+	encodedErrorResponse := encodeResponse(stsErrorResponse)
+	writeResponse(w, stsError.HTTPStatusCode, encodedErrorResponse, mimeXML)
+}
+
+// STSError structure
+type STSError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+// STSErrorResponse - error response format
+type STSErrorResponse struct {
+	XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ ErrorResponse" json:"-"`
+	Error   struct {
+		Type    string `xml:"Type"`
+		Code    string `xml:"Code"`
+		Message string `xml:"Message"`
+	} `xml:"Error"`
+	RequestID string `xml:"RequestId"`
+}
+
+// STSErrorCode type of error status.
+type STSErrorCode int
+
+// Error codes, non exhaustive list - http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithSAML.html
+const (
+	ErrSTSNone STSErrorCode = iota
+	ErrSTSMissingParameter
+	ErrSTSInvalidParameterValue
+	ErrSTSClientGrantsExpiredToken
+	ErrSTSInvalidClientGrantsToken
+	ErrSTSMalformedPolicyDocument
+	ErrSTSNotInitialized
+	ErrSTSInternalError
+)
+
+// error code to STSError structure, these fields carry respective
+// descriptions for all the error responses.
+var stsErrCodeResponse = map[STSErrorCode]STSError{
+	ErrSTSMissingParameter: {
+		Code:           "MissingParameter",
+		Description:    "A required parameter for the specified action is not supplied.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSInvalidParameterValue: {
+		Code:           "InvalidParameterValue",
+		Description:    "An invalid or out-of-range value was supplied for the input parameter.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSClientGrantsExpiredToken: {
+		Code:           "ExpiredToken",
+		Description:    "The client grants that was passed is expired or is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSInvalidClientGrantsToken: {
+		Code:           "InvalidClientGrantsToken",
+		Description:    "The client grants token that was passed could not be validated by Minio.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSMalformedPolicyDocument: {
+		Code:           "MalformedPolicyDocument",
+		Description:    "The request was rejected because the policy document was malformed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSNotInitialized: {
+		Code:           "STSNotInitialized",
+		Description:    "STS API not initialized, please try again.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSTSInternalError: {
+		Code:           "InternalError",
+		Description:    "We encountered an internal error generating credentials, please try again.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+}
+
+// getSTSError provides STS Error for input STS error code.
+func getSTSError(code STSErrorCode) STSError {
+	return stsErrCodeResponse[code]
+}
+
+// getErrorResponse gets in standard error and resource value and
+// provides a encodable populated response values
+func getSTSErrorResponse(err STSError) STSErrorResponse {
+	errRsp := STSErrorResponse{}
+	errRsp.Error.Code = err.Code
+	errRsp.Error.Message = err.Description
+	errRsp.RequestID = "3L137"
+	return errRsp
+}

cmd/sts-handlers.go

@@ -0,0 +1,186 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/iam/validator"
+)
+
+const (
+	// STS API version.
+	stsAPIVersion = "2011-06-15"
+)
+
+// stsAPIHandlers implements and provides http handlers for AWS STS API.
+type stsAPIHandlers struct{}
+
+// registerSTSRouter - registers AWS STS compatible APIs.
+func registerSTSRouter(router *mux.Router) {
+	// Initialize STS.
+	sts := &stsAPIHandlers{}
+
+	// STS Router
+	stsRouter := router.NewRoute().PathPrefix("/").Subrouter()
+
+	// AssumeRoleWithClientGrants
+	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithClientGrants)).
+		Queries("Action", "AssumeRoleWithClientGrants").
+		Queries("Token", "{Token:.*}")
+
+}
+
+// AssumedRoleUser - The identifiers for the temporary security credentials that
+// the operation returns. Please also see https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumedRoleUser
+type AssumedRoleUser struct {
+	// The ARN of the temporary security credentials that are returned from the
+	// AssumeRole action. For more information about ARNs and how to use them in
+	// policies, see IAM Identifiers (http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html)
+	// in Using IAM.
+	//
+	// Arn is a required field
+	Arn string
+
+	// A unique identifier that contains the role ID and the role session name of
+	// the role that is being assumed. The role ID is generated by AWS when the
+	// role is created.
+	//
+	// AssumedRoleId is a required field
+	AssumedRoleID string `xml:"AssumeRoleId"`
+	// contains filtered or unexported fields
+}
+
+// AssumeRoleWithClientGrantsResponse contains the result of successful AssumeRoleWithClientGrants request.
+type AssumeRoleWithClientGrantsResponse struct {
+	XMLName          xml.Name           `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithClientGrantsResponse" json:"-"`
+	Result           ClientGrantsResult `xml:"AssumeRoleWithClientGrantsResult"`
+	ResponseMetadata struct {
+		RequestID string `xml:"RequestId,omitempty"`
+	} `xml:"ResponseMetadata,omitempty"`
+}
+
+// ClientGrantsResult - Contains the response to a successful AssumeRoleWithClientGrants
+// request, including temporary credentials that can be used to make Minio API requests.
+type ClientGrantsResult struct {
+	// The identifiers for the temporary security credentials that the operation
+	// returns.
+	AssumedRoleUser AssumedRoleUser `xml:",omitempty"`
+
+	// The intended audience (also known as client ID) of the web identity token.
+	// This is traditionally the client identifier issued to the application that
+	// requested the client grants.
+	Audience string `xml:",omitempty"`
+
+	// The temporary security credentials, which include an access key ID, a secret
+	// access key, and a security (or session) token.
+	//
+	// Note: The size of the security token that STS APIs return is not fixed. We
+	// strongly recommend that you make no assumptions about the maximum size. As
+	// of this writing, the typical size is less than 4096 bytes, but that can vary.
+	// Also, future updates to AWS might require larger sizes.
+	Credentials auth.Credentials `xml:",omitempty"`
+
+	// A percentage value that indicates the size of the policy in packed form.
+	// The service rejects any policy with a packed size greater than 100 percent,
+	// which means the policy exceeded the allowed space.
+	PackedPolicySize int `xml:",omitempty"`
+
+	// The issuing authority of the web identity token presented. For OpenID Connect
+	// ID tokens, this contains the value of the iss field. For OAuth 2.0 access tokens,
+	// this contains the value of the ProviderId parameter that was passed in the
+	// AssumeRoleWithClientGrants request.
+	Provider string `xml:",omitempty"`
+
+	// The unique user identifier that is returned by the identity provider.
+	// This identifier is associated with the Token that was submitted
+	// with the AssumeRoleWithClientGrants call. The identifier is typically unique to
+	// the user and the application that acquired the ClientGrantsToken (pairwise identifier).
+	// For OpenID Connect ID tokens, this field contains the value returned by the identity
+	// provider as the token's sub (Subject) claim.
+	SubjectFromToken string `xml:",omitempty"`
+}
+
+// AssumeRoleWithClientGrants - implementation of AWS STS extension API supporting
+// OAuth2.0 client grants.
+//
+// Eg:-
+//    $ curl https://minio:9000/?Action=AssumeRoleWithClientGrants&Token=<jwt>
+func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AssumeRoleWithClientGrants")
+
+	if globalIAMValidators == nil {
+		writeSTSErrorResponse(w, ErrSTSNotInitialized)
+		return
+	}
+
+	// NOTE: this API only accepts JWT tokens.
+	v, err := globalIAMValidators.Get("jwt")
+	if err != nil {
+		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		return
+	}
+
+	vars := mux.Vars(r)
+	m, err := v.Validate(vars["Token"], r.URL.Query().Get("DurationSeconds"))
+	if err != nil {
+		switch err {
+		case validator.ErrTokenExpired:
+			writeSTSErrorResponse(w, ErrSTSClientGrantsExpiredToken)
+		case validator.ErrInvalidDuration:
+			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		default:
+			logger.LogIf(ctx, err)
+			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		}
+		return
+	}
+
+	secret := globalServerConfig.GetCredential().SecretKey
+	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, ErrSTSInternalError)
+		return
+	}
+
+	// JWT has requested a custom claim with policy value set.
+	// This is a Minio STS API specific value, this value should
+	// be set and configured on your identity provider as part of
+	// JWT custom claims.
+	var policyName string
+	if v, ok := m["policy"]; ok {
+		policyName, _ = v.(string)
+	}
+
+	// Set the newly generated credentials.
+	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, ErrSTSInternalError)
+		return
+	}
+
+	encodedSuccessResponse := encodeResponse(&AssumeRoleWithClientGrantsResponse{
+		Result: ClientGrantsResult{Credentials: cred},
+	})
+
+	writeSuccessResponseXML(w, encodedSuccessResponse)
+}

cmd/test-utils_test.go

@@ -354,10 +354,16 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
 	globalMinioPort = port
 	globalMinioAddr = getEndpointsLocalAddr(testServer.Disks)
 
-	globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks)
+	globalConfigSys = NewConfigSys()
+
+	globalIAMSys = NewIAMSys()
+	globalIAMSys.Init(objLayer)
 
-	// Create new policy system.
 	globalPolicySys = NewPolicySys()
+	globalPolicySys.Init(objLayer)
+
+	globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks)
+	globalNotificationSys.Init(objLayer)
 
 	return testServer
 }
@@ -465,6 +471,11 @@ func resetGlobalCacheObjectAPI() {
 	globalCacheObjectAPI = nil
 }
 
+// sets globalIAMSys to `nil`.
+func resetGlobalIAMSys() {
+	globalIAMSys = nil
+}
+
 // Resets all the globals used modified in tests.
 // Resetting ensures that the changes made to globals by one test doesn't affect others.
 func resetTestGlobals() {
@@ -488,14 +499,16 @@ func resetTestGlobals() {
 	resetGlobalHealState()
 	//Reset global disk cache flags
 	resetGlobalCacheEnvs()
-	//set globalCacheObjectAPI to nil
+	// Reset globalCacheObjectAPI to nil
 	resetGlobalCacheObjectAPI()
+	// Reset globalIAMSys to `nil`
+	resetGlobalIAMSys()
 }
 
 // Configure the server for the test run.
 func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) {
 	// Initialize server config.
-	if err = newConfig(obj); err != nil {
+	if err = newSrvConfig(obj); err != nil {
 		return err
 	}
 
@@ -1620,11 +1633,13 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro
 		return xl.storageDisks
 	}
 
-	// Create new notification system.
-	globalNotificationSys = NewNotificationSys(globalServerConfig, endpoints)
+	globalConfigSys = NewConfigSys()
+
+	globalIAMSys = NewIAMSys()
+	globalIAMSys.Init(xl)
 
-	// Create new policy system.
 	globalPolicySys = NewPolicySys()
+	globalNotificationSys = NewNotificationSys(globalServerConfig, endpoints)
 
 	return xl, nil
 }
@@ -1894,9 +1909,11 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 	}
 	bucketFS, fsAPIRouter, err := initAPIHandlerTest(objLayer, endpoints)
 	if err != nil {
-		t.Fatalf("Initialzation of API handler tests failed: <ERROR> %s", err)
+		t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)
 	}
 
+	globalIAMSys = NewIAMSys()
+	globalIAMSys.Init(objLayer)
 	// initialize the server and obtain the credentials and root.
 	// credentials are necessary to sign the HTTP request.
 	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {

cmd/typed-errors.go

@@ -76,3 +76,12 @@ var errBucketAlreadyExists = errors.New("Your previous request to create the nam
 
 // error returned for a negative actual size.
 var errInvalidDecompressedSize = errors.New("Invalid Decompressed Size")
+
+// error returned in IAM subsystem when user doesn't exist.
+var errNoSuchUser = errors.New("Specified user does not exist")
+
+// error returned in IAM subsystem when policy doesn't exist.
+var errNoSuchPolicy = errors.New("Specified canned policy does not exist")
+
+// error returned when access is denied.
+var errAccessDenied = errors.New("Do not have enough permissions to access this resource")

cmd/utils.go

@@ -218,18 +218,26 @@ func startProfiler(profilerType, dirPath string) (interface {
 		Stop()
 	}
 
-	// Enable profiler, supported types are [cpu, mem, block].
+	var profilerFileName string
+
+	// Enable profiler and set the name of the file that pkg/pprof
+	// library creates to store profiling data.
 	switch profilerType {
 	case "cpu":
 		profiler = profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "cpu.pprof"
 	case "mem":
 		profiler = profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "mem.pprof"
 	case "block":
 		profiler = profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "block.pprof"
 	case "mutex":
 		profiler = profile.Start(profile.MutexProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "mutex.pprof"
 	case "trace":
 		profiler = profile.Start(profile.TraceProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "trace.out"
 	default:
 		return nil, errors.New("profiler type unknown")
 	}
@@ -237,7 +245,7 @@ func startProfiler(profilerType, dirPath string) (interface {
 	return &profilerWrapper{
 		stopFn: profiler.Stop,
 		pathFn: func() string {
-			return filepath.Join(dirPath, profilerType+".pprof")
+			return filepath.Join(dirPath, profilerFileName)
 		},
 	}, nil
 }
@@ -391,7 +399,7 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
 	reqInfo := &logger.ReqInfo{
 		RequestID:  w.Header().Get(responseRequestIDKey),
 		RemoteHost: handlers.GetSourceIP(r),
-		UserAgent:  r.Header.Get("user-agent"),
+		UserAgent:  r.UserAgent(),
 		API:        api,
 		BucketName: bucket,
 		ObjectName: object,

cmd/web-handlers.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -44,7 +45,9 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -71,8 +74,9 @@ type ServerInfoRep struct {
 
 // ServerInfo - get server info.
 func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, reply *ServerInfoRep) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 	host, err := os.Hostname()
 	if err != nil {
@@ -93,6 +97,14 @@ func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, rep
 
 	reply.MinioVersion = Version
 	reply.MinioGlobalInfo = getGlobalInfo()
+	// If ENV creds are not set and incoming user is not owner
+	// disable changing credentials.
+	// TODO: fix this in future and allow changing user credentials.
+	v, ok := reply.MinioGlobalInfo["isEnvCreds"].(bool)
+	if ok && !v {
+		reply.MinioGlobalInfo["isEnvCreds"] = !owner
+	}
+
 	reply.MinioMemory = mem
 	reply.MinioPlatform = platform
 	reply.MinioRuntime = goruntime
@@ -112,8 +124,9 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *AuthArgs, reply *S
 	if objectAPI == nil {
 		return toJSONError(errServerNotInitialized)
 	}
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, _, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 	reply.StorageInfo = objectAPI.StorageInfo(context.Background())
 	reply.UIVersion = browser.UIVersion
@@ -131,8 +144,13 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
 	if objectAPI == nil {
 		return toJSONError(errServerNotInitialized)
 	}
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	// TODO: Allow MakeBucket in future.
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	// Check if bucket is a reserved bucket name or invalid.
@@ -179,8 +197,13 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
 	if objectAPI == nil {
 		return toJSONError(errServerNotInitialized)
 	}
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	// TODO: Allow DeleteBucket in future.
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	ctx := context.Background()
@@ -234,10 +257,11 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
 	if web.CacheAPI() != nil {
 		listBuckets = web.CacheAPI().ListBuckets
 	}
-	authErr := webRequestAuthenticate(r)
-	if authErr != nil {
+
+	if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
 		return toJSONError(authErr)
 	}
+
 	// If etcd, dns federation configured list buckets from etcd.
 	if globalDNSConfig != nil {
 		dnsBuckets, err := globalDNSConfig.List()
@@ -303,43 +327,76 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
 	if objectAPI == nil {
 		return toJSONError(errServerNotInitialized)
 	}
+
 	listObjects := objectAPI.ListObjects
 	if web.CacheAPI() != nil {
 		listObjects = web.CacheAPI().ListObjects
 	}
 
-	// Check if anonymous (non-owner) has access to download objects.
-	readable := globalPolicySys.IsAllowed(policy.Args{
-		Action:          policy.GetObjectAction,
-		BucketName:      args.BucketName,
-		ConditionValues: getConditionValues(r, ""),
-		IsOwner:         false,
-		ObjectName:      args.Prefix + "/",
-	})
-	// Check if anonymous (non-owner) has access to upload objects.
-	writable := globalPolicySys.IsAllowed(policy.Args{
-		Action:          policy.PutObjectAction,
-		BucketName:      args.BucketName,
-		ConditionValues: getConditionValues(r, ""),
-		IsOwner:         false,
-		ObjectName:      args.Prefix + "/",
-	})
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		if authErr == errNoAuthToken {
+			// Check if anonymous (non-owner) has access to download objects.
+			readable := globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.GetObjectAction,
+				BucketName:      args.BucketName,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+				ObjectName:      args.Prefix + "/",
+			})
 
-	if authErr := webRequestAuthenticate(r); authErr != nil {
-		if authErr == errAuthentication {
+			// Check if anonymous (non-owner) has access to upload objects.
+			writable := globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.PutObjectAction,
+				BucketName:      args.BucketName,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+				ObjectName:      args.Prefix + "/",
+			})
+
+			reply.Writable = writable
+			if !readable {
+				// Error out if anonymous user (non-owner) has no access to download or upload objects
+				if !writable {
+					return errAccessDenied
+				}
+				// return empty object list if access is write only
+				return nil
+			}
+		} else {
 			return toJSONError(authErr)
 		}
+	}
+
+	// For authenticated users apply IAM policy.
+	if authErr == nil {
+		readable := globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.GetObjectAction),
+			BucketName:      args.BucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         owner,
+			ObjectName:      args.Prefix + "/",
+		})
+
+		writable := globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.PutObjectAction),
+			BucketName:      args.BucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         owner,
+			ObjectName:      args.Prefix + "/",
+		})
 
 		reply.Writable = writable
 		if !readable {
 			// Error out if anonymous user (non-owner) has no access to download or upload objects
 			if !writable {
-				return errAuthentication
+				return errAccessDenied
 			}
 			// return empty object list if access is write only
 			return nil
 		}
-
 	}
 
 	lo, err := listObjects(context.Background(), args.BucketName, args.Prefix, args.Marker, slashSeparator, 1000)
@@ -398,8 +455,10 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs,
 	if web.CacheAPI() != nil {
 		listObjects = web.CacheAPI().ListObjects
 	}
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 
 	if args.BucketName == "" || len(args.Objects) == 0 {
@@ -418,12 +477,34 @@ next:
 				}
 			}
 
+			if !globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     claims.Subject,
+				Action:          iampolicy.Action(policy.DeleteObjectAction),
+				BucketName:      args.BucketName,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         owner,
+				ObjectName:      objectName,
+			}) {
+				return toJSONError(errAccessDenied)
+			}
+
 			if err = deleteObject(nil, objectAPI, web.CacheAPI(), args.BucketName, objectName, r); err != nil {
 				break next
 			}
 			continue
 		}
 
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.DeleteObjectAction),
+			BucketName:      args.BucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         owner,
+			ObjectName:      objectName,
+		}) {
+			return toJSONError(errAccessDenied)
+		}
+
 		// For directories, list the contents recursively and remove.
 		marker := ""
 		for {
@@ -486,8 +567,12 @@ type GenerateAuthReply struct {
 }
 
 func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, reply *GenerateAuthReply) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 	cred, err := auth.GetNewCredentials()
 	if err != nil {
@@ -514,12 +599,14 @@ type SetAuthReply struct {
 
 // SetAuth - Set accessKey and secretKey credentials.
 func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *SetAuthReply) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 
 	// If creds are set through ENV disallow changing credentials.
-	if globalIsEnvCreds || globalWORMEnabled {
+	// TODO: Multi-user credentials also cannot be changed from browser.
+	if globalIsEnvCreds || globalWORMEnabled || !owner {
 		return toJSONError(errChangeCredNotAllowed)
 	}
 
@@ -551,7 +638,10 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
 			reply.PeerErrMsgs[host.String()] = err.Error()
 		}
 	} else {
-		reply.Token = newAuthToken()
+		reply.Token, err = authenticateWeb(creds.AccessKey, creds.SecretKey)
+		if err != nil {
+			return toJSONError(err)
+		}
 		reply.UIVersion = browser.UIVersion
 	}
 
@@ -567,8 +657,12 @@ type GetAuthReply struct {
 
 // GetAuth - return accessKey and secretKey credentials.
 func (web *webAPIHandlers) GetAuth(r *http.Request, args *WebGenericArgs, reply *GetAuthReply) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 	creds := globalServerConfig.GetCredential()
 	reply.AccessKey = creds.AccessKey
@@ -585,11 +679,19 @@ type URLTokenReply struct {
 
 // CreateURLToken creates a URL token (short-lived) for GET requests.
 func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs, reply *URLTokenReply) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 
 	creds := globalServerConfig.GetCredential()
+	if !owner {
+		var ok bool
+		creds, ok = globalIAMSys.GetUser(claims.Subject)
+		if !ok {
+			return toJSONError(errInvalidAccessKeyID)
+		}
+	}
 
 	token, err := authenticateURL(creds.AccessKey, creds.SecretKey)
 	if err != nil {
@@ -603,6 +705,10 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs,
 
 // Upload - file upload handler.
 func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "WebUpload")
+
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		writeWebErrorResponse(w, errServerNotInitialized)
@@ -617,18 +723,34 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	bucket := vars["bucket"]
 	object := vars["object"]
 
-	if authErr := webRequestAuthenticate(r); authErr != nil {
-		if authErr == errAuthentication {
-			writeWebErrorResponse(w, errAuthentication)
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		if authErr == errNoAuthToken {
+			// Check if anonymous (non-owner) has access to upload objects.
+			if !globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.PutObjectAction,
+				BucketName:      bucket,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+				ObjectName:      object,
+			}) {
+				writeWebErrorResponse(w, errAuthentication)
+				return
+			}
+		} else {
+			writeWebErrorResponse(w, authErr)
 			return
 		}
+	}
 
-		// Check if anonymous (non-owner) has access to upload objects.
-		if !globalPolicySys.IsAllowed(policy.Args{
-			Action:          policy.PutObjectAction,
+	// For authenticated users apply IAM policy.
+	if authErr == nil {
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.PutObjectAction),
 			BucketName:      bucket,
 			ConditionValues: getConditionValues(r, ""),
-			IsOwner:         false,
+			IsOwner:         owner,
 			ObjectName:      object,
 		}) {
 			writeWebErrorResponse(w, errAuthentication)
@@ -688,29 +810,43 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	opts := ObjectOptions{}
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
-		if _, err = objectAPI.GetObjectInfo(context.Background(), bucket, object, opts); err == nil {
+		if _, err = objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil {
 			writeWebErrorResponse(w, errMethodNotAllowed)
 			return
 		}
 	}
 
-	objInfo, err := putObject(context.Background(), bucket, object, hashReader, metadata, opts)
+	objInfo, err := putObject(ctx, bucket, object, hashReader, metadata, opts)
 	if err != nil {
 		writeWebErrorResponse(w, err)
 		return
 	}
 
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
+	if err != nil {
+		host, port = "", ""
+	}
+
 	// Notify object created event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedPut,
-		BucketName: bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
+		EventName:    event.ObjectCreatedPut,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 }
 
 // Download - file download handler.
 func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "WebDownload")
+
+	defer logger.AuditLog(ctx, w, r)
+
 	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
@@ -723,13 +859,34 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	object := vars["object"]
 	token := r.URL.Query().Get("token")
 
-	if !isAuthTokenValid(token) {
-		// Check if anonymous (non-owner) has access to download objects.
-		if !globalPolicySys.IsAllowed(policy.Args{
-			Action:          policy.GetObjectAction,
+	claims, owner, authErr := webTokenAuthenticate(token)
+	if authErr != nil {
+		if authErr == errNoAuthToken {
+			// Check if anonymous (non-owner) has access to download objects.
+			if !globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.GetObjectAction,
+				BucketName:      bucket,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+				ObjectName:      object,
+			}) {
+				writeWebErrorResponse(w, errAuthentication)
+				return
+			}
+		} else {
+			writeWebErrorResponse(w, authErr)
+			return
+		}
+	}
+
+	// For authenticated users apply IAM policy.
+	if authErr == nil {
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.GetObjectAction),
 			BucketName:      bucket,
 			ConditionValues: getConditionValues(r, ""),
-			IsOwner:         false,
+			IsOwner:         owner,
 			ObjectName:      object,
 		}) {
 			writeWebErrorResponse(w, errAuthentication)
@@ -744,7 +901,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 		getObjectInfo = web.CacheAPI().GetObjectInfo
 		getObject = web.CacheAPI().GetObject
 	}
-	objInfo, err := getObjectInfo(context.Background(), bucket, object, opts)
+	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		writeWebErrorResponse(w, err)
 		return
@@ -814,7 +971,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	// Add content disposition.
 	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(object)))
 
-	if err = getObject(context.Background(), bucket, object, 0, -1, httpWriter, "", opts); err != nil {
+	if err = getObject(ctx, bucket, object, 0, -1, httpWriter, "", opts); err != nil {
 		httpWriter.Close()
 		if objInfo.IsCompressed() {
 			wg.Wait()
@@ -832,6 +989,24 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 		// Wait for decompression go-routine to retire.
 		wg.Wait()
 	}
+
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
+	if err != nil {
+		host, port = "", ""
+	}
+
+	// Notify object accessed via a GET request.
+	sendEvent(eventArgs{
+		EventName:    event.ObjectAccessedGet,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
+	})
 }
 
 // DownloadZipArgs - Argument for downloading a bunch of files as a zip file.
@@ -845,20 +1020,22 @@ type DownloadZipArgs struct {
 
 // Takes a list of objects and creates a zip file that sent as the response body.
 func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
+	if err != nil {
+		host, port = "", ""
+	}
+
+	ctx := newContext(r, w, "WebDownloadZip")
+	defer logger.AuditLog(ctx, w, r)
+
 	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		writeWebErrorResponse(w, errServerNotInitialized)
 		return
 	}
-	getObject := objectAPI.GetObject
-	if web.CacheAPI() != nil {
-		getObject = web.CacheAPI().GetObject
-	}
-	listObjects := objectAPI.ListObjects
-	if web.CacheAPI() != nil {
-		listObjects = web.CacheAPI().ListObjects
-	}
+
 	// Auth is done after reading the body to accommodate for anonymous requests
 	// when bucket policy is enabled.
 	var args DownloadZipArgs
@@ -870,14 +1047,37 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 	}
 
 	token := r.URL.Query().Get("token")
-	if !isAuthTokenValid(token) {
+	claims, owner, authErr := webTokenAuthenticate(token)
+	if authErr != nil {
+		if authErr == errNoAuthToken {
+			for _, object := range args.Objects {
+				// Check if anonymous (non-owner) has access to download objects.
+				if !globalPolicySys.IsAllowed(policy.Args{
+					Action:          policy.GetObjectAction,
+					BucketName:      args.BucketName,
+					ConditionValues: getConditionValues(r, ""),
+					IsOwner:         false,
+					ObjectName:      pathJoin(args.Prefix, object),
+				}) {
+					writeWebErrorResponse(w, errAuthentication)
+					return
+				}
+			}
+		} else {
+			writeWebErrorResponse(w, authErr)
+			return
+		}
+	}
+
+	// For authenticated users apply IAM policy.
+	if authErr == nil {
 		for _, object := range args.Objects {
-			// Check if anonymous (non-owner) has access to download objects.
-			if !globalPolicySys.IsAllowed(policy.Args{
-				Action:          policy.GetObjectAction,
+			if !globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     claims.Subject,
+				Action:          iampolicy.Action(policy.GetObjectAction),
 				BucketName:      args.BucketName,
 				ConditionValues: getConditionValues(r, ""),
-				IsOwner:         false,
+				IsOwner:         owner,
 				ObjectName:      pathJoin(args.Prefix, object),
 			}) {
 				writeWebErrorResponse(w, errAuthentication)
@@ -886,8 +1086,18 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	getObject := objectAPI.GetObject
+	if web.CacheAPI() != nil {
+		getObject = web.CacheAPI().GetObject
+	}
+	listObjects := objectAPI.ListObjects
+	if web.CacheAPI() != nil {
+		listObjects = web.CacheAPI().ListObjects
+	}
+
 	archive := zip.NewWriter(w)
 	defer archive.Close()
+
 	getObjectInfo := objectAPI.GetObjectInfo
 	if web.CacheAPI() != nil {
 		getObjectInfo = web.CacheAPI().GetObjectInfo
@@ -897,7 +1107,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 	for _, object := range args.Objects {
 		// Writes compressed object file to the response.
 		zipit := func(objectName string) error {
-			info, err := getObjectInfo(context.Background(), args.BucketName, objectName, opts)
+			info, err := getObjectInfo(ctx, args.BucketName, objectName, opts)
 			if err != nil {
 				return err
 			}
@@ -970,7 +1180,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 				}
 			}
 			httpWriter := ioutil.WriteOnClose(writer)
-			if err = getObject(context.Background(), args.BucketName, objectName, 0, length, httpWriter, "", opts); err != nil {
+			if err = getObject(ctx, args.BucketName, objectName, 0, length, httpWriter, "", opts); err != nil {
 				httpWriter.Close()
 				if info.IsCompressed() {
 					// Wait for decompression go-routine to retire.
@@ -988,6 +1198,19 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 				// Wait for decompression go-routine to retire.
 				wg.Wait()
 			}
+
+			// Notify object accessed via a GET request.
+			sendEvent(eventArgs{
+				EventName:    event.ObjectAccessedGet,
+				BucketName:   args.BucketName,
+				Object:       info,
+				ReqParams:    extractReqParams(r),
+				RespElements: extractRespElements(w),
+				UserAgent:    r.UserAgent(),
+				Host:         host,
+				Port:         port,
+			})
+
 			return nil
 		}
 
@@ -1041,8 +1264,12 @@ func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolic
 		return toJSONError(errServerNotInitialized)
 	}
 
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	bucketPolicy, err := objectAPI.GetBucketPolicy(context.Background(), args.BucketName)
@@ -1090,8 +1317,12 @@ func (web *webAPIHandlers) ListAllBucketPolicies(r *http.Request, args *ListAllB
 		return toJSONError(errServerNotInitialized)
 	}
 
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	bucketPolicy, err := objectAPI.GetBucketPolicy(context.Background(), args.BucketName)
@@ -1137,8 +1368,12 @@ func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolic
 		return toJSONError(errServerNotInitialized)
 	}
 
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	policyType := miniogopolicy.BucketPolicy(args.Policy)
@@ -1215,27 +1450,37 @@ type PresignedGetRep struct {
 
 // PresignedGET - returns presigned-Get url.
 func (web *webAPIHandlers) PresignedGet(r *http.Request, args *PresignedGetArgs, reply *PresignedGetRep) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	var creds auth.Credentials
+	if !owner {
+		var ok bool
+		creds, ok = globalIAMSys.GetUser(claims.Subject)
+		if !ok {
+			return toJSONError(errInvalidAccessKeyID)
+		}
+	} else {
+		creds = globalServerConfig.GetCredential()
 	}
 
+	region := globalServerConfig.GetRegion()
 	if args.BucketName == "" || args.ObjectName == "" {
 		return &json2.Error{
 			Message: "Bucket and Object are mandatory arguments.",
 		}
 	}
+
 	reply.UIVersion = browser.UIVersion
-	reply.URL = presignedGet(args.HostName, args.BucketName, args.ObjectName, args.Expiry)
+	reply.URL = presignedGet(args.HostName, args.BucketName, args.ObjectName, args.Expiry, creds, region)
 	return nil
 }
 
 // Returns presigned url for GET method.
-func presignedGet(host, bucket, object string, expiry int64) string {
-	cred := globalServerConfig.GetCredential()
-	region := globalServerConfig.GetRegion()
-
-	accessKey := cred.AccessKey
-	secretKey := cred.SecretKey
+func presignedGet(host, bucket, object string, expiry int64, creds auth.Credentials, region string) string {
+	accessKey := creds.AccessKey
+	secretKey := creds.SecretKey
 
 	date := UTCNow()
 	dateStr := date.Format(iso8601Format)

cmd/web-handlers_test.go

@@ -343,7 +343,7 @@ func testDeleteBucketWebHandler(obj ObjectLayer, instanceType string, t TestErrH
 		{"minio", false, token, "specified bucket minio does not exist"},
 		{bucketName, false, token, ""},
 		{bucketName, true, token, "Bucket not empty"},
-		{bucketName, false, "", "Authentication failed"},
+		{bucketName, false, "", "JWT token missing"},
 	}
 
 	for _, test := range testCases {
@@ -1294,7 +1294,7 @@ func testWebGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestE
 		},
 	}
 
-	if err = savePolicyConfig(obj, bucketName, bucketPolicy); err != nil {
+	if err = savePolicyConfig(context.Background(), obj, bucketName, bucketPolicy); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -1394,7 +1394,7 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t
 		},
 	}
 
-	if err = savePolicyConfig(obj, bucketName, bucketPolicy); err != nil {
+	if err = savePolicyConfig(context.Background(), obj, bucketName, bucketPolicy); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 

cmd/xl-sets.go

@@ -174,7 +174,7 @@ func (s *xlSets) reInitDisks(refFormat *formatXLV3, storageDisks []StorageAPI, f
 // any given sets.
 func (s *xlSets) connectDisksWithQuorum() {
 	var onlineDisks int
-	for onlineDisks < (len(s.endpoints)/2)+1 {
+	for onlineDisks < len(s.endpoints)/2 {
 		for _, endpoint := range s.endpoints {
 			if s.isConnected(endpoint) {
 				continue
@@ -499,7 +499,7 @@ func (s *xlSets) ListObjectsV2(ctx context.Context, bucket, prefix, continuation
 
 // SetBucketPolicy persist the new policy on the bucket.
 func (s *xlSets) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
-	return savePolicyConfig(s, bucket, policy)
+	return savePolicyConfig(ctx, s, bucket, policy)
 }
 
 // GetBucketPolicy will return a policy on a bucket
@@ -1202,8 +1202,14 @@ func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun bool) (re
 	for _, endpoint := range s.endpoints {
 		var foundBefore bool
 		for _, v := range res.Before.Drives {
-			if v.Endpoint == endpoint.String() {
-				foundBefore = true
+			if endpoint.IsLocal {
+				if v.Endpoint == endpoint.Path {
+					foundBefore = true
+				}
+			} else {
+				if v.Endpoint == endpoint.String() {
+					foundBefore = true
+				}
 			}
 		}
 		if !foundBefore {
@@ -1215,8 +1221,14 @@ func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun bool) (re
 		}
 		var foundAfter bool
 		for _, v := range res.After.Drives {
-			if v.Endpoint == endpoint.String() {
-				foundAfter = true
+			if endpoint.IsLocal {
+				if v.Endpoint == endpoint.Path {
+					foundAfter = true
+				}
+			} else {
+				if v.Endpoint == endpoint.String() {
+					foundAfter = true
+				}
 			}
 		}
 		if !foundAfter {

cmd/xl-v1-bucket.go

@@ -244,7 +244,6 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
 			err := disk.DeleteVol(bucket)
 
 			if err != nil {
-				logger.LogIf(ctx, err)
 				dErrs[index] = err
 				return
 			}
@@ -277,7 +276,7 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
 
 // SetBucketPolicy sets policy on bucket
 func (xl xlObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
-	return savePolicyConfig(xl, bucket, policy)
+	return savePolicyConfig(ctx, xl, bucket, policy)
 }
 
 // GetBucketPolicy will get policy on bucket

cmd/xl-v1-healing.go

@@ -36,29 +36,6 @@ func (xl xlObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRes
 	return madmin.HealResultItem{}, NotImplemented{}
 }
 
-// checks for bucket if it exists in writeQuorum number of disks, this call
-// is only used by healBucket().
-func checkBucketExistsInQuorum(ctx context.Context, storageDisks []StorageAPI, bucketName string) (err error) {
-	var wg = &sync.WaitGroup{}
-
-	errs := make([]error, len(storageDisks))
-	// Prepare object creation in a all disks
-	for index, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		wg.Add(1)
-		go func(index int, disk StorageAPI) {
-			defer wg.Done()
-			_, errs[index] = disk.StatVol(bucketName)
-		}(index, disk)
-	}
-	wg.Wait()
-
-	readQuorum := len(storageDisks) / 2
-	return reduceWriteQuorumErrs(ctx, errs, nil, readQuorum)
-}
-
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
@@ -67,13 +44,6 @@ func (xl xlObjects) HealBucket(ctx context.Context, bucket string, dryRun bool)
 
 	storageDisks := xl.getDisks()
 
-	// Check if bucket doesn't exist in writeQuorum number of disks, if quorum
-	// number of disks returned that bucket does not exist we quickly return
-	// and do not proceed to heal.
-	if err = checkBucketExistsInQuorum(ctx, storageDisks, bucket); err != nil {
-		return results, err
-	}
-
 	// get write quorum for an object
 	writeQuorum := len(storageDisks)/2 + 1
 

cmd/xl-v1-metadata.go

@@ -218,6 +218,8 @@ func (m xlMetaV1) ToObjectInfo(bucket, object string) ObjectInfo {
 		ContentEncoding: m.Meta["content-encoding"],
 	}
 
+	objInfo.backendType = BackendErasure
+
 	// Extract etag from metadata.
 	objInfo.ETag = extractETag(m.Meta)
 

cmd/xl-v1-multipart.go

@@ -343,7 +343,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	// Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete.
 	defer xl.deleteObject(ctx, minioMetaTmpBucket, tmpPart, writeQuorum, false)
 	if data.Size() > 0 || data.Size() == -1 {
-		if pErr := xl.prepareFile(ctx, minioMetaTmpBucket, tmpPartPath, data.Size(), onlineDisks, xlMeta.Erasure.BlockSize, xlMeta.Erasure.DataBlocks, writeQuorum); err != nil {
+		if pErr := xl.prepareFile(ctx, minioMetaTmpBucket, tmpPartPath, data.Size(), onlineDisks, xlMeta.Erasure.BlockSize, xlMeta.Erasure.DataBlocks, writeQuorum); pErr != nil {
 			return pi, toObjectErr(pErr, bucket, object)
 
 		}
@@ -359,7 +359,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	switch size := data.Size(); {
 	case size == 0:
 		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
-	case size == -1 || size > blockSizeV1:
+	case size == -1 || size >= blockSizeV1:
 		buffer = xl.bp.Get()
 		defer xl.bp.Put(buffer)
 	case size < blockSizeV1:
@@ -385,7 +385,6 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	// Should return IncompleteBody{} error when reader has fewer bytes
 	// than specified in request header.
 	if n < data.Size() {
-		logger.LogIf(ctx, IncompleteBody{})
 		return pi, IncompleteBody{}
 	}
 
@@ -655,7 +654,6 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 				PartNumber: part.PartNumber,
 				GotETag:    part.ETag,
 			}
-			logger.LogIf(ctx, invp)
 			return oi, invp
 		}
 
@@ -666,17 +664,11 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 				ExpETag:    currentXLMeta.Parts[partIdx].ETag,
 				GotETag:    part.ETag,
 			}
-			logger.LogIf(ctx, invp)
 			return oi, invp
 		}
 
 		// All parts except the last part has to be atleast 5MB.
 		if (i < len(parts)-1) && !isMinAllowedPartSize(currentXLMeta.Parts[partIdx].ActualSize) {
-			logger.LogIf(ctx, PartTooSmall{
-				PartNumber: part.PartNumber,
-				PartSize:   currentXLMeta.Parts[partIdx].ActualSize,
-				PartETag:   part.ETag,
-			})
 			return oi, PartTooSmall{
 				PartNumber: part.PartNumber,
 				PartSize:   currentXLMeta.Parts[partIdx].ActualSize,

cmd/xl-v1-object.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"path"
 	"strconv"
+	"strings"
 	"sync"
 
 	"github.com/minio/minio/cmd/logger"
@@ -447,19 +448,105 @@ func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string, op
 	return info, nil
 }
 
-// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
-func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
-	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object)
-
-	// get Quorum for this object
-	readQuorum, _, err := objectQuorumFromMeta(ctx, xl, metaArr, errs)
-	if err != nil {
-		return objInfo, err
+func (xl xlObjects) isObjectCorrupted(metaArr []xlMetaV1, errs []error) (validMeta xlMetaV1, ok bool) {
+	// We can consider an object data not reliable
+	// when xl.json is not found in read quorum disks.
+	var notFoundXLJSON int
+	for _, readErr := range errs {
+		if readErr == errFileNotFound {
+			notFoundXLJSON++
+		}
 	}
 
-	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
-		return objInfo, reducedErr
+	for _, m := range metaArr {
+		if !m.IsValid() {
+			continue
+		}
+		validMeta = m
+		break
+	}
+
+	// Return if the object is indeed corrupted.
+	return validMeta, len(xl.getDisks())-notFoundXLJSON < validMeta.Erasure.DataBlocks
+}
+
+const xlCorruptedSuffix = ".CORRUPTED"
+
+// Renames the corrupted object and makes it visible.
+func renameCorruptedObject(ctx context.Context, bucket, object string, validMeta xlMetaV1, disks []StorageAPI, errs []error) {
+	writeQuorum := validMeta.Erasure.DataBlocks + 1
+
+	// Move all existing objects into corrupted suffix.
+	rename(ctx, disks, bucket, object, bucket, object+xlCorruptedSuffix, true, writeQuorum, []error{errFileNotFound})
+
+	tempObj := mustGetUUID()
+
+	// Get all the disks which do not have the file.
+	var cdisks = make([]StorageAPI, len(disks))
+	for i, merr := range errs {
+		if merr == errFileNotFound {
+			cdisks[i] = disks[i]
+		}
+	}
+
+	for _, disk := range cdisks {
+		if disk == nil {
+			continue
+		}
+
+		// Write empty part file on missing disks.
+		disk.AppendFile(minioMetaTmpBucket, pathJoin(tempObj, "part.1"), []byte{})
+
+		// Write algorithm hash for empty part file.
+		alg := validMeta.Erasure.Checksums[0].Algorithm.New()
+		alg.Write([]byte{})
+
+		// Update the checksums and part info.
+		validMeta.Erasure.Checksums[0] = ChecksumInfo{
+			Name:      validMeta.Erasure.Checksums[0].Name,
+			Algorithm: validMeta.Erasure.Checksums[0].Algorithm,
+			Hash:      alg.Sum(nil),
+		}
+		validMeta.Parts[0] = objectPartInfo{
+			Number: 1,
+			Name:   "part.1",
+		}
+
+		// Write the `xl.json` with the newly calculated metadata.
+		writeXLMetadata(ctx, disk, minioMetaTmpBucket, tempObj, validMeta)
+	}
+
+	// Finally rename all the parts into their respective locations.
+	rename(ctx, cdisks, minioMetaTmpBucket, tempObj, bucket, object+xlCorruptedSuffix, true, writeQuorum, []error{errFileNotFound})
+}
+
+// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
+func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
+	disks := xl.getDisks()
+
+	// Read metadata associated with the object from all disks.
+	metaArr, errs := readAllXLMetadata(ctx, disks, bucket, object)
+
+	var readQuorum int
+	// Having read quorum means we have xl.json in at least N/2 disks.
+	if !strings.HasSuffix(object, xlCorruptedSuffix) {
+		if validMeta, ok := xl.isObjectCorrupted(metaArr, errs); ok {
+			renameCorruptedObject(ctx, bucket, object, validMeta, disks, errs)
+			// Return err file not found since we renamed now the corrupted object
+			return objInfo, errFileNotFound
+		}
+
+		// Not a corrupted object, attempt to get readquorum properly.
+		readQuorum, _, err = objectQuorumFromMeta(ctx, xl, metaArr, errs)
+		if err != nil {
+			return objInfo, err
+		}
+
+	} else {
+
+		// If this is a corrupted object, change read quorum to N/2 disks
+		// so it will be visible to users, so they can delete it.
+		readQuorum = len(xl.getDisks()) / 2
 	}
 
 	// List all the file commit ids from parts metadata.
@@ -604,8 +691,9 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 
-		// Rename the successfully written temporary object to final location.
-		if _, err = rename(ctx, xl.getDisks(), minioMetaTmpBucket, tempObj, bucket, object, true, writeQuorum, nil); err != nil {
+		// Rename the successfully written temporary object to final location. Ignore errFileAccessDenied
+		// error because it means that the target object dir exists and we want to be close to S3 specification.
+		if _, err = rename(ctx, xl.getDisks(), minioMetaTmpBucket, tempObj, bucket, object, true, writeQuorum, []error{errFileAccessDenied}); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 
@@ -896,13 +984,13 @@ func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (er
 		return err
 	}
 
-	if !xl.isObject(bucket, object) && !xl.isObjectDir(bucket, object) {
-		return ObjectNotFound{bucket, object}
-	}
-
 	var writeQuorum int
 	var isObjectDir = hasSuffix(object, slashSeparator)
 
+	if isObjectDir && !xl.isObjectDir(bucket, object) {
+		return toObjectErr(errFileNotFound, bucket, object)
+	}
+
 	if isObjectDir {
 		writeQuorum = len(xl.getDisks())/2 + 1
 	} else {

cmd/xl-v1-utils.go

@@ -305,7 +305,7 @@ func readXLMeta(ctx context.Context, disk StorageAPI, bucket string, object stri
 	// Reads entire `xl.json`.
 	xlMetaBuf, err := disk.ReadAll(bucket, path.Join(object, xlMetaJSONFile))
 	if err != nil {
-		if err != errFileNotFound {
+		if err != errFileNotFound && err != errVolumeNotFound {
 			logger.GetReqInfo(ctx).AppendTags("disk", disk.String())
 			logger.LogIf(ctx, err)
 		}

docs/bucket/notifications/README.md

@@ -409,7 +409,7 @@ This notification target supports two formats: _namespace_ and _access_.
 
 When the _namespace_ format is used, Minio synchronizes objects in the bucket with entries in a hash. For each entry, the key is formatted as "bucketName/objectName" for an object that exists in the bucket, and the value is the JSON-encoded event data about the operation that created/replaced the object in Minio. When objects are updated or deleted, the corresponding entry in the hash is also updated or deleted.
 
-When the _access_ format is used, Minio appends events to a list using [RPUSH](https://redis.io/commands/rpush). Each item in the list is a JSON encoded list with two items, where the first item is a timestamp string, and second item is a JSON object containing evnet data about the operation that happened in the bucket. No entries appended to the list are updated or deleted by Minio in this format.
+When the _access_ format is used, Minio appends events to a list using [RPUSH](https://redis.io/commands/rpush). Each item in the list is a JSON encoded list with two items, where the first item is a timestamp string, and the second item is a JSON object containing event data about the operation that happened in the bucket. No entries appended to the list are updated or deleted by Minio in this format.
 
 The steps below show how to use this notification target in `namespace` and `access` format.
 

docs/config/README.md

@@ -1,9 +1,8 @@
-# Minio Server `config.json` (v27) Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
-
-Minio server stored all its configuration data in `${HOME}/.minio/config.json` file by default.For all releases beyond minio.RELEASE.2017-08-05T00-00-53Z, the configuration data will be migrated to the Minio's backend directory. All configuration changes can be made using [`mc admin config` get/set commands](https://github.com/minio/mc/blob/master/docs/minio-admin-complete-guide.md). Following sections provide detailed explanation of each fields and how to customize them. A complete example of `config.json` is available [here](https://raw.githubusercontent.com/minio/minio/master/docs/config/config.sample.json)
+# Minio Server Config Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
 
 ## Configuration Directory
-The default configuration directory is `${HOME}/.minio`. Till the release `RELEASE.2018-08-02T23-11-36Z` , Minio server configuration file (`config.json`) was stored in the configuration directory. However for releases beyond `RELEASE.2018-08-18T03-49-57Z`, the configuration file (only), has been migrated to the backend storage directory.
+
+The default configuration directory is `${HOME}/.minio`. Till the release `RELEASE.2018-08-02T23-11-36Z`, Minio server configuration file (`config.json`) was stored in the configuration directory. However for releases beyond `RELEASE.2018-08-18T03-49-57Z`, the configuration file (only), has been migrated to the storage back-end (storage back-end is the directory passed to Minio server while starting the server).
 
 You can override the default configuration directory using `--config-dir` command-line option. Please note that this won't have an effect on the `config.json` file as it is always stored on the backend storage, along with data. Minio server generates a new `config.json` with auto-generated access credentials when its started for the first time.
 
@@ -12,6 +11,7 @@ minio server --config-dir /etc/minio /data
 ```
 
 ### Certificate Directory
+
 TLS certificates are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to Minio server with TLS](https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls).
 
 Following is the directory structure for Minio server with TLS certificates.
@@ -25,14 +25,23 @@ $ tree ~/.minio
 │   └── public.crt
 ```
 
-### Configuration Fields
-#### Editing Configuration Fields
-For releases after minio.RELEASE.2017-08-05T00-00-53Z, configuration changes can be done using [mc]((https://docs.minio.io/docs/minio-client-quickstart-guide))
+
+### Accessing configuration file
+
+All configuration changes can be made using [`mc admin config` get/set commands](https://github.com/minio/mc/blob/master/docs/minio-admin-complete-guide.md). Following sections provide brief explanation of fields and how to customize them. A complete example of `config.json` is available [here](https://raw.githubusercontent.com/minio/minio/master/docs/config/config.sample.json)
+
+#### Editing configuration file fields
+
+
+
 ##### Get current configuration for Minio deployment
+
 ```sh
 $ mc admin config get myminio/ > /tmp/myconfig
 ```
+
 ##### Set current configuration for Minio deployment
+
 ```sh
 $ mc admin config set myminio < /tmp/myconfig
 ```
@@ -40,11 +49,13 @@ $ mc admin config set myminio < /tmp/myconfig
 The `mc admin` config API will evolve soon to be able to configure specific fields using get/set commands.
 
 #### Version
+
 |Field|Type|Description|
 |:---|:---|:---|
-|``version``|_string_| `version` determines the configuration file format. Any older version will be automatically be migrated to the latest version upon startup. [DO NOT EDIT THIS FIELD MANUALLY]|
+|``version``|_string_| `version` determines the configuration file format. Any older version will automatically be migrated to the latest version upon startup. [DO NOT EDIT THIS FIELD MANUALLY]|
 
 #### Credential
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``credential``| | Auth credential for object storage and web access.|
@@ -60,6 +71,7 @@ minio server /data
 ```
 
 #### Region
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``region``| _string_ | `region` describes the physical location of the server. By default it is blank. You may override this field with `MINIO_REGION` environment variable. If you are unsure leave it unset.|
@@ -72,6 +84,7 @@ minio server /data
 ```
 
 #### Worm
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``worm``| _string_ | Enable this to turn on Write-Once-Read-Many. By default it is set to `off`. You may override this field with ``MINIO_WORM`` environment variable.|
@@ -84,6 +97,7 @@ minio server /data
 ```
 
 ### Storage Class
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``storageclass``| | Set storage class for configurable data and parity, as per object basis.|
@@ -93,6 +107,7 @@ minio server /data
 By default, parity for objects with standard storage class is set to `N/2`, and parity for objects with reduced redundancy storage class objects is set to `2`. Read more about storage class support in Minio server [here](https://github.com/minio/minio/blob/master/docs/erasure/storage-class/README.md).
 
 ### Cache
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``drives``| _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled|
@@ -101,6 +116,7 @@ By default, parity for objects with standard storage class is set to `N/2`, and
 |``maxuse`` | _int_ | Percentage of disk available to cache |
 
 #### Notify
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``notify``| |Notify enables bucket notification events for lambda computing via the following targets.|
@@ -117,22 +133,27 @@ By default, parity for objects with standard storage class is set to `N/2`, and
 ## Environment only settings
 
 #### Browser
+
 Enable or disable access to web UI. By default it is set to `on`. You may override this field with ``MINIO_BROWSER`` environment variable.
 
 Example:
+
 ```sh
 export MINIO_BROWSER=off
 minio server /data
 ```
 
 ### Domain
-By default, Minio supports path-style requests which look like http://mydomain.com/bucket/object. MINIO_DOMAIN environmental variable is used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the mattched pattern `$1` is used as bucket and the path is used as object. More information on path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html)
 
+By default, Minio supports path-style requests that are of the format http://mydomain.com/bucket/object. MINIO_DOMAIN environment variable is used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the matched pattern `$1` is used as bucket and the path is used as object. More information on path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html)
 Example:
+
 ```sh
 export MINIO_DOMAIN=mydomain.com
 minio server /data
 ```
 
 ## Explore Further
+
 * [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide)
+* [Configure Minio Server with TLS](https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls)

docs/config/config.sample.json

@@ -1,136 +1,189 @@
 {
-    "version": "29",
-    "credential": {
-        "accessKey": "USWUXHGYZQYFYFFIT3RE",
-        "secretKey": "MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03"
-    },
-    "region": "us-east-1",
-    "worm": "off",
-    "storageclass": {
-        "standard": "",
-        "rrs": ""
-    },
-    "cache": {
-        "drives": [],
-        "expiry": 90,
-        "exclude": [],
-        "maxuse": 80
-    },
-    "usage": {
-        "interval": "3h"
-    },
-    "notify": {
-        "amqp": {
-            "1": {
-                "enable": true,
-                "url": "amqp://myuser:mypassword@localhost:5672",
-                "exchange": "bucketevents",
-                "routingKey": "bucketlogs",
-                "exchangeType": "fanout",
-                "deliveryMode": 0,
-                "mandatory": false,
-                "immediate": false,
-                "durable": false,
-                "internal": false,
-                "noWait": false,
-                "autoDeleted": false
-            }
-        },
-        "nats": {
-            "1": {
-                "enable": true,
-                "address": "0.0.0.0:4222",
-                "subject": "bucketevents",
-                "username": "yourusername",
-                "password": "yoursecret",
-                "token": "",
-                "secure": false,
-                "pingInterval": 0,
-                "streaming": {
-                    "enable": false,
-                    "clusterID": "",
-                    "clientID": "",
-                    "async": false,
-                    "maxPubAcksInflight": 0
-                }
-            }
-        },
-        "elasticsearch": {
-            "1": {
-                "enable": true,
-                "format": "namespace",
-                "url": "http://127.0.0.1:9200",
-                "index": "bucketevents"
-            }
-        },
-        "redis": {
-            "1": {
-                "enable": true,
-                "format": "namespace",
-                "address": "127.0.0.1:6379",
-                "password": "yoursecret",
-                "key": "bucketevents"
-            }
-        },
-        "postgresql": {
-            "1": {
-                "enable": true,
-                "format": "namespace",
-                "connectionString": "",
-                "table": "bucketevents",
-                "host": "127.0.0.1",
-                "port": "5432",
-                "user": "postgres",
-                "password": "mypassword",
-                "database": "bucketevents_db"
-            }
-        },
-        "kafka": {
-            "1": {
-                "enable": true,
-                "brokers": ["localhost:9092"],
-                "topic": "bucketevents",
-                "tls" : {
-                    "enable" : true,
-                    "skipVerify" : false,
-                    "clientAuth" : 0
-                },
-                "sasl" : {
-                    "enable" : true,
-                    "username" : "kafka",
-                    "password" : "kafkapasswd"
-                }
-            }
-        },
-        "webhook": {
-            "1": {
-                "enable": true,
-                "endpoint": "http://requestb.in/1525x9n1"
-            }
-        },
-        "mysql": {
-            "1": {
-                "enable": true,
-                "format": "namespace",
-                "dsnString": "",
-                "table": "minio_images",
-                "host": "172.17.0.1",
-                "port": "3306",
-                "user": "root",
-                "password": "password",
-                "database": "miniodb"
-            }
-        },
-        "mqtt": {
-            "1": {
-                "enable": false,
-                "broker": "tcp://localhost:1883",
-                "topic": "minio",
-                "qos": 1,
-                "clientId": "minio",
-                "username": "",
-                "password": ""
-            }
-        }
-    }
+	"version": "31",
+	"credential": {
+		"accessKey": "36J9X8EZI4KEV1G7EHXA",
+		"secretKey": "ECk2uqOoNqvtJIMQ3WYugvmNPL_-zm3WcRqP5vUM",
+		"expiration": "1970-01-01T00:00:00Z",
+		"status": "enabled"
+	},
+	"region": "",
+	"worm": "off",
+	"storageclass": {
+		"standard": "",
+		"rrs": ""
+	},
+	"cache": {
+		"drives": [],
+		"expiry": 90,
+		"maxuse": 80,
+		"exclude": []
+	},
+	"kms": {
+		"vault": {
+			"endpoint": "",
+			"auth": {
+				"type": "",
+				"approle": {
+					"id": "",
+					"secret": ""
+				}
+			},
+			"key-id": {
+				"name": "",
+				"version": 0
+			}
+		}
+	},
+	"notify": {
+		"amqp": {
+			"1": {
+				"enable": false,
+				"url": "",
+				"exchange": "",
+				"routingKey": "",
+				"exchangeType": "",
+				"deliveryMode": 0,
+				"mandatory": false,
+				"immediate": false,
+				"durable": false,
+				"internal": false,
+				"noWait": false,
+				"autoDeleted": false
+			}
+		},
+		"elasticsearch": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"url": "",
+				"index": ""
+			}
+		},
+		"kafka": {
+			"1": {
+				"enable": false,
+				"brokers": null,
+				"topic": "",
+				"tls": {
+					"enable": false,
+					"skipVerify": false,
+					"clientAuth": 0
+				},
+				"sasl": {
+					"enable": false,
+					"username": "",
+					"password": ""
+				}
+			}
+		},
+		"mqtt": {
+			"1": {
+				"enable": false,
+				"broker": "",
+				"topic": "",
+				"qos": 0,
+				"clientId": "",
+				"username": "",
+				"password": "",
+				"reconnectInterval": 0,
+				"keepAliveInterval": 0
+			}
+		},
+		"mysql": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"dsnString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
+		"nats": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"subject": "",
+				"username": "",
+				"password": "",
+				"token": "",
+				"secure": false,
+				"pingInterval": 0,
+				"streaming": {
+					"enable": false,
+					"clusterID": "",
+					"clientID": "",
+					"async": false,
+					"maxPubAcksInflight": 0
+				}
+			}
+		},
+		"postgresql": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"connectionString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
+		"redis": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"address": "",
+				"password": "",
+				"key": ""
+			}
+		},
+		"webhook": {
+			"1": {
+				"enable": false,
+				"endpoint": ""
+			}
+		}
+	},
+	"logger": {
+		"console": {
+			"enabled": true
+		},
+		"http": {
+			"target1": {
+				"enabled": false,
+				"endpoint": "https://username:password@example.com/api"
+			}
+		}
+	},
+	"compress": {
+		"enabled": false,
+		"extensions": [
+			".txt",
+			".log",
+			".csv",
+			".json"
+		],
+		"mime-types": [
+			"text/csv",
+			"text/plain",
+			"application/json"
+		]
+	},
+	"openid": {
+		"jwks": {
+			"url": null
+		}
+	},
+	"policy": {
+		"opa": {
+			"url": null,
+			"authToken": ""
+		}
+	}
 }