Reload etcd users and policies properly (#6694)

Currently there was a bug in how we reload users and policies
which leads to users/policies going missing due to wrong path
construction.

Fixes #6693

.travis.yml

@@ -1,10 +1,4 @@
 go_import_path: github.com/minio/minio
-sudo: required
-
-services:
- - docker
-
-dist: trusty
 
 language: go
 
@@ -18,9 +12,11 @@ branches:
 matrix:
   include:
     - os: linux
+      dist: trusty
+      sudo: required
       env:
         - ARCH=x86_64
-      go: 1.10.3
+      go: 1.10.4
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
@@ -29,9 +25,17 @@ matrix:
         - make verify
         - make coverage
         - cd browser && yarn && yarn test && cd ..
+    - os: windows
+      env:
+        - ARCH=x86_64
+      go: 1.10.4
+      script:
+        - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
+        - for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
+        - bash buildscripts/go-coverage.sh
 
 before_install:
-- nvm install stable
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
 
 after_success:
 - bash <(curl -s https://codecov.io/bash)

Makefile

@@ -13,7 +13,7 @@ checks:
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 getdeps:
-	@echo "Installing golint" && go get -u github.com/golang/lint/golint
+	@echo "Installing golint" && go get -u golang.org/x/lint/golint
 	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
 	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
 	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell

appveyor.yml

@@ -1,55 +0,0 @@
-# version format
-version: "{build}"
-
-# Operating system (build VM template)
-os: Windows Server 2012 R2
-
-# Platform.
-platform: x64
-
-clone_folder: c:\gopath\src\github.com\minio\minio
-
-# Environment variables
-environment:
-  GOPATH: c:\gopath
-  GOROOT: c:\go110
-
-# scripts that run after cloning repository
-install:
-  - set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
-  - go version
-  - go env
-  - python --version
-
-# To run your custom scripts instead of automatic MSBuild
-build_script:
-  # Compile
-  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
-  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
-  - appveyor AddCompilationMessage "Starting Compile"
-  - cd c:\gopath\src\github.com\minio\minio
-  - go run buildscripts/gen-ldflags.go > temp.txt
-  - set /p BUILD_LDFLAGS=<temp.txt
-  - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
-  - appveyor AddCompilationMessage "Compile Success"
-
-# To run your custom scripts instead of automatic tests
-test_script:
-  # Unit tests
-  - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
-  - mkdir build\coverage
-  - for /f "" %%G in ('go list github.com/minio/minio/... ^| find /i /v "browser/"') do ( go test -v -timeout 20m -race %%G )
-  - go test -v -timeout 20m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
-  - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
-
-after_test:
-  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
-  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
-  - ps: Push-AppveyorArtifact build\coverage\coverage.html
-  # Upload coverage report.
-  - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
-  - pip install codecov
-  - codecov -X gcov -f "build\coverage\coverage.txt"
-
-# to disable deployment
-deploy: off

buildscripts/go-coverage.sh

@@ -4,7 +4,7 @@ set -e
 echo "" > coverage.txt
 
 for d in $(go list ./... | grep -v browser); do
-    go test -coverprofile=profile.out -covermode=atomic "$d"
+    go test -v -coverprofile=profile.out -covermode=atomic "$d"
     if [ -f profile.out ]; then
         cat profile.out >> coverage.txt
         rm profile.out

cmd/acl-handlers.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -56,6 +57,8 @@ type accessControlPolicy struct {
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketACL")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -103,6 +106,8 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectACL")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]

cmd/admin-handlers.go

@@ -36,6 +36,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/quick"
 	"github.com/tidwall/gjson"
@@ -43,7 +44,7 @@ import (
 )
 
 const (
-	maxConfigJSONSize = 256 * 1024 // 256KiB
+	maxEConfigJSONSize = 262272
 )
 
 // Type-safe query params.
@@ -105,7 +106,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 	// of read-quorum availability.
 	uptime, err := getPeerUptimes(globalAdminPeers)
 	if err != nil {
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		logger.LogIf(context.Background(), err)
 		return
 	}
@@ -597,7 +598,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	password := config.GetCredential().SecretKey
-	econfigData, err := madmin.EncryptServerConfigData(password, configData)
+	econfigData, err := madmin.EncryptData(password, configData)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
@@ -681,7 +682,7 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	password := config.GetCredential().SecretKey
-	econfigData, err := madmin.EncryptServerConfigData(password, []byte(newConfigStr))
+	econfigData, err := madmin.EncryptData(password, []byte(newConfigStr))
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
@@ -702,6 +703,348 @@ func toAdminAPIErrCode(err error) APIErrorCode {
 	}
 }
 
+// RemoveUser - DELETE /minio/admin/v1/remove-user?accessKey=<access_key>
+func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveUser")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+	}
+}
+
+// ListUsers - GET /minio/admin/v1/list-users
+func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListUsers")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	allCredentials, err := globalIAMSys.ListUsers()
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(allCredentials)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrInternalError, r.URL)
+		return
+	}
+
+	password := globalServerConfig.GetCredential().SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrInternalError, r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// SetUserStatus - PUT /minio/admin/v1/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
+func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetUserStatus")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+	status := vars["status"]
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalServerConfig.GetCredential().AccessKey {
+		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		return
+	}
+
+	if err := globalIAMSys.SetUserStatus(accessKey, madmin.AccountStatus(status)); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+}
+
+// AddUser - PUT /minio/admin/v1/add-user?accessKey=<access_key>
+func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddUser")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalServerConfig.GetCredential().AccessKey {
+		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
+		return
+	}
+
+	password := globalServerConfig.GetCredential().SecretKey
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
+		return
+	}
+
+	var uinfo madmin.UserInfo
+	if err = json.Unmarshal(configBytes, &uinfo); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
+		return
+	}
+
+	if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+}
+
+// ListCannedPolicies - GET /minio/admin/v1/list-canned-policies
+func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListCannedPolicies")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	policies, err := globalIAMSys.ListCannedPolicies()
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(policies); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// RemoveCannedPolicy - DELETE /minio/admin/v1/remove-canned-policy?name=<policy_name>
+func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveCannedPolicy")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["name"]
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	if err := globalIAMSys.DeleteCannedPolicy(policyName); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+}
+
+// AddCannedPolicy - PUT /minio/admin/v1/add-canned-policy?name=<policy_name>
+func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddCannedPolicy")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars["name"]
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	// Error out if Content-Length is missing.
+	if r.ContentLength <= 0 {
+		writeErrorResponseJSON(w, ErrMissingContentLength, r.URL)
+		return
+	}
+
+	// Error out if Content-Length is beyond allowed size.
+	if r.ContentLength > maxBucketPolicySize {
+		writeErrorResponseJSON(w, ErrEntityTooLarge, r.URL)
+		return
+	}
+
+	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponseJSON(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	// Version in policy must not be empty
+	if iamPolicy.Version == "" {
+		writeErrorResponseJSON(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	if err = globalIAMSys.SetCannedPolicy(policyName, *iamPolicy); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+}
+
+// SetUserPolicy - PUT /minio/admin/v1/set-user-policy?accessKey=<access_key>&name=<policy_name>
+func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetUserPolicy")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+	policyName := vars["name"]
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	// Custom IAM policies not allowed for admin user.
+	if accessKey == globalServerConfig.GetCredential().AccessKey {
+		writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
+		return
+	}
+
+	if err := globalIAMSys.SetUserPolicy(accessKey, policyName); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+	}
+}
+
 // SetConfigHandler - PUT /minio/admin/v1/config
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigHandler")
@@ -726,22 +1069,14 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	// Read configuration bytes from request body.
-	configBuf := make([]byte, maxConfigJSONSize+1)
-	n, err := io.ReadFull(r.Body, configBuf)
-	if err == nil {
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
 		return
 	}
-	if err != io.ErrUnexpectedEOF {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
-		return
-	}
 
 	password := globalServerConfig.GetCredential().SecretKey
-	configBytes, err := madmin.DecryptServerConfigData(password, bytes.NewReader(configBuf[:n]))
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
@@ -757,8 +1092,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	var config serverConfig
-	err = json.Unmarshal(configBytes, &config)
-	if err != nil {
+	if err = json.Unmarshal(configBytes, &config); err != nil {
 		logger.LogIf(ctx, err)
 		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
 		return
@@ -780,6 +1114,11 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	if err = config.TestNotificationTargets(); err != nil {
+		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
+		return
+	}
+
 	if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return
@@ -860,7 +1199,7 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 			writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
 			return
 		}
-		elem, dErr := madmin.DecryptServerConfigData(password, bytes.NewBuffer([]byte(encryptedElem)))
+		elem, dErr := madmin.DecryptData(password, bytes.NewBuffer([]byte(encryptedElem)))
 		if dErr != nil {
 			logger.LogIf(ctx, dErr)
 			writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
@@ -920,11 +1259,10 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// UpdateCredsHandler - POST /minio/admin/v1/config/credential
+// UpdateAdminCredsHandler - POST /minio/admin/v1/config/credential
 // ----------
-// Update credentials in a minio server. In a distributed setup,
-// update all the servers in the cluster.
-func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
+// Update admin credentials in a minio server
+func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 	r *http.Request) {
 
 	ctx := newContext(r, w, "UpdateCredentialsHandler")
@@ -938,7 +1276,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 
 	// Avoid setting new credentials when they are already passed
 	// by the environment. Deny if WORM is enabled.
-	if globalIsEnvCreds {
+	if globalIsEnvCreds || globalWORMEnabled {
 		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
 		return
 	}
@@ -950,22 +1288,14 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Read configuration bytes from request body.
-	configBuf := make([]byte, maxConfigJSONSize+1)
-	n, err := io.ReadFull(r.Body, configBuf)
-	if err == nil {
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
 		return
 	}
-	if err != io.ErrUnexpectedEOF {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
-		return
-	}
 
 	password := globalServerConfig.GetCredential().SecretKey
-	configBytes, err := madmin.DecryptServerConfigData(password, bytes.NewReader(configBuf[:n]))
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
@@ -982,7 +1312,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 
 	creds, err := auth.CreateCredentials(req.AccessKey, req.SecretKey)
 	if err != nil {
-		writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return
 	}
 
@@ -993,6 +1323,9 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 	// Update local credentials in memory.
 	globalServerConfig.SetCredential(creds)
 
+	// Set active creds.
+	globalActiveCred = creds
+
 	if err = saveServerConfig(ctx, objectAPI, globalServerConfig); err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return

cmd/admin-handlers_test.go

@@ -38,172 +38,184 @@ import (
 
 var (
 	configJSON = []byte(`{
-	"version": "30",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "",
-	"worm": "off",
-	"storageclass": {
-		"standard": "",
-		"rrs": ""
-	},
-	"cache": {
-		"drives": [],
-		"expiry": 90,
-		"maxuse": 80,
-		"exclude": []
-	},
-	"kms": {
-		"vault": {
-			"endpoint": "",
-			"auth": {
-				"type": "",
-				"approle": {
-					"id": "",
-					"secret": ""
-				}
-			},
-			"key-id": {
-				"name": "",
-				"version": 0
-			}
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"deliveryMode": 0,
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"url": "",
-				"index": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": "",
-				"tls" : {
-					"enable" : false,
-					"skipVerify" : false,
-					"clientAuth" : 0
-				},
-				"sasl" : {
-					"enable" : false,
-					"username" : "",
-					"password" : ""
-				}
-			}
-		},
-		"mqtt": {
-			"1": {
-				"enable": false,
-				"broker": "",
-				"topic": "",
-				"qos": 0,
-				"clientId": "",
-				"username": "",
-				"password": "",
-				"reconnectInterval": 0,
-				"keepAliveInterval": 0
-			}
-		},
-		"mysql": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"dsnString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	},
-	"logger": {
-		"console": {
-			"enabled": true
-		},
-		"http": {
-			"target1": {
-				"enabled": false,
-				"endpoint": "https://username:password@example.com/api"
-			}
-		}
-	},
-	"compress": {
-		"enabled": false,
-        	"extensions":[".txt",".log",".csv",".json"],
-        	"mime-types":["text/csv","text/plain","application/json"]
-	}
-}`)
+  "version": "31",
+  "credential": {
+    "accessKey": "minio",
+    "secretKey": "minio123"
+  },
+  "region": "us-east-1",
+  "worm": "off",
+  "storageclass": {
+    "standard": "",
+    "rrs": ""
+  },
+  "cache": {
+    "drives": [],
+    "expiry": 90,
+    "maxuse": 80,
+    "exclude": []
+  },
+  "kms": {
+    "vault": {
+      "endpoint": "",
+      "auth": {
+        "type": "",
+        "approle": {
+          "id": "",
+          "secret": ""
+        }
+      },
+      "key-id": {
+        "name": "",
+        "version": 0
+      }
+    }
+  },
+  "notify": {
+    "amqp": {
+      "1": {
+        "enable": false,
+        "url": "",
+        "exchange": "",
+        "routingKey": "",
+        "exchangeType": "",
+        "deliveryMode": 0,
+        "mandatory": false,
+        "immediate": false,
+        "durable": false,
+        "internal": false,
+        "noWait": false,
+        "autoDeleted": false
+      }
+    },
+    "elasticsearch": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "url": "",
+        "index": ""
+      }
+    },
+    "kafka": {
+      "1": {
+        "enable": false,
+        "brokers": null,
+        "topic": "",
+        "tls": {
+          "enable": false,
+          "skipVerify": false,
+          "clientAuth": 0
+        },
+        "sasl": {
+          "enable": false,
+          "username": "",
+          "password": ""
+        }
+      }
+    },
+    "mqtt": {
+      "1": {
+        "enable": false,
+        "broker": "",
+        "topic": "",
+        "qos": 0,
+        "clientId": "",
+        "username": "",
+        "password": "",
+        "reconnectInterval": 0,
+        "keepAliveInterval": 0
+      }
+    },
+    "mysql": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "dsnString": "",
+        "table": "",
+        "host": "",
+        "port": "",
+        "user": "",
+        "password": "",
+        "database": ""
+      }
+    },
+    "nats": {
+      "1": {
+        "enable": false,
+        "address": "",
+        "subject": "",
+        "username": "",
+        "password": "",
+        "token": "",
+        "secure": false,
+        "pingInterval": 0,
+        "streaming": {
+          "enable": false,
+          "clusterID": "",
+          "clientID": "",
+          "async": false,
+          "maxPubAcksInflight": 0
+        }
+      }
+    },
+    "postgresql": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "connectionString": "",
+        "table": "",
+        "host": "",
+        "port": "",
+        "user": "",
+        "password": "",
+        "database": ""
+      }
+    },
+    "redis": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "address": "",
+        "password": "",
+        "key": ""
+      }
+    },
+    "webhook": {
+      "1": {
+        "enable": false,
+        "endpoint": ""
+      }
+    }
+  },
+  "logger": {
+    "console": {
+      "enabled": true
+    },
+    "http": {
+      "1": {
+        "enabled": false,
+        "endpoint": "https://username:password@example.com/api"
+      }
+    }
+  },
+  "compress": {
+    "enabled": false,
+    "extensions":[".txt",".log",".csv",".json"],
+    "mime-types":["text/csv","text/plain","application/json"]
+  },
+  "openid": {
+    "jwks": {
+      "url": ""
+    }
+  },
+  "policy": {
+    "opa": {
+      "url": "",
+      "authToken": ""
+    }
+  }
+}
+`)
 )
 
 // adminXLTestBed - encapsulates subsystems that need to be setup for
@@ -485,6 +497,8 @@ func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*htt
 
 	// Set body
 	req.Body = ioutil.NopCloser(bytes.NewReader(body))
+	req.ContentLength = int64(len(body))
+
 	// Set sha-sum header
 	req.Header.Set("X-Amz-Content-Sha256", getSHA256Hash(body))
 
@@ -615,7 +629,7 @@ func TestServiceSetCreds(t *testing.T) {
 			t.Fatalf("JSONify err: %v", err)
 		}
 
-		ebody, err := madmin.EncryptServerConfigData(credentials.SecretKey, body)
+		ebody, err := madmin.EncryptData(credentials.SecretKey, body)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -718,7 +732,7 @@ func TestSetConfigHandler(t *testing.T) {
 	queryVal.Set("config", "")
 
 	password := globalServerConfig.GetCredential().SecretKey
-	econfigJSON, err := madmin.EncryptServerConfigData(password, configJSON)
+	econfigJSON, err := madmin.EncryptData(password, configJSON)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -738,7 +752,7 @@ func TestSetConfigHandler(t *testing.T) {
 	// Check that a very large config file returns an error.
 	{
 		// Make a large enough config string
-		invalidCfg := []byte(strings.Repeat("A", maxConfigJSONSize+1))
+		invalidCfg := []byte(strings.Repeat("A", maxEConfigJSONSize+1))
 		req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
 			int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
 		if err != nil {
@@ -768,7 +782,7 @@ func TestSetConfigHandler(t *testing.T) {
 		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
-			!strings.Contains(respBody, "JSON configuration provided has objects with duplicate keys") {
+			!strings.Contains(respBody, "JSON configuration provided is of incorrect format") {
 			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
 		}
 	}

cmd/admin-router.go

@@ -53,12 +53,14 @@ func registerAdminRouter(router *mux.Router) {
 	// Info operations
 	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 
-	/// Heal operations
+	if globalIsDistXL || globalIsXL {
+		/// Heal operations
 
-	// Heal processing endpoint.
-	adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		// Heal processing endpoint.
+		adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+	}
 
 	// Profiling operations
 	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
@@ -68,7 +70,7 @@ func registerAdminRouter(router *mux.Router) {
 	/// Config operations
 
 	// Update credentials
-	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateCredentialsHandler))
+	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
 	// Get config
 	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 	// Set config
@@ -78,4 +80,31 @@ func registerAdminRouter(router *mux.Router) {
 	adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
 	// Set config keys/values
 	adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
+
+	// -- IAM APIs --
+
+	// Add policy IAM
+	adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+
+	// Add user IAM
+	adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+	adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
+		Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
+	adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
+		Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+
+	// Remove policy IAM
+	adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+
+	// Remove user IAM
+	adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+
+	// List users
+	adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+
+	// List policies
+	adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+
+	// If none of the routes match.
+	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandler))
 }

cmd/api-datatypes.go

@@ -25,90 +25,6 @@ const (
 	responseRequestIDKey = "x-amz-request-id"
 )
 
-// CSVFileHeaderInfo -Can be either USE IGNORE OR NONE, defines what to do with
-// the first row
-type CSVFileHeaderInfo string
-
-// Constants for file header info.
-const (
-	CSVFileHeaderInfoNone   CSVFileHeaderInfo = "NONE"
-	CSVFileHeaderInfoIgnore                   = "IGNORE"
-	CSVFileHeaderInfoUse                      = "USE"
-)
-
-// SelectCompressionType - ONLY GZIP is supported
-type SelectCompressionType string
-
-// Constants for compression types under select API.
-const (
-	SelectCompressionNONE SelectCompressionType = "NONE"
-	SelectCompressionGZIP                       = "GZIP"
-	SelectCompressionBZIP                       = "BZIP2"
-)
-
-// CSVQuoteFields - Can be either Always or AsNeeded
-type CSVQuoteFields string
-
-// Constants for csv quote styles.
-const (
-	CSVQuoteFieldsAlways   CSVQuoteFields = "Always"
-	CSVQuoteFieldsAsNeeded                = "AsNeeded"
-)
-
-// QueryExpressionType - Currently can only be SQL
-type QueryExpressionType string
-
-// Constants for expression type.
-const (
-	QueryExpressionTypeSQL QueryExpressionType = "SQL"
-)
-
-// JSONType determines json input serialization type.
-type JSONType string
-
-// Constants for JSONTypes.
-const (
-	JSONDocumentType JSONType = "Document"
-	JSONLinesType             = "Lines"
-)
-
-// ObjectSelectRequest - represents the input select body
-type ObjectSelectRequest struct {
-	XMLName            xml.Name `xml:"SelectObjectContentRequest" json:"-"`
-	Expression         string
-	ExpressionType     QueryExpressionType
-	InputSerialization struct {
-		CompressionType SelectCompressionType
-		Parquet         *struct{}
-		CSV             *struct {
-			FileHeaderInfo       CSVFileHeaderInfo
-			RecordDelimiter      string
-			FieldDelimiter       string
-			QuoteCharacter       string
-			QuoteEscapeCharacter string
-			Comments             string
-		}
-		JSON *struct {
-			Type JSONType
-		}
-	}
-	OutputSerialization struct {
-		CSV *struct {
-			QuoteFields          CSVQuoteFields
-			RecordDelimiter      string
-			FieldDelimiter       string
-			QuoteCharacter       string
-			QuoteEscapeCharacter string
-		}
-		JSON *struct {
-			RecordDelimiter string
-		}
-	}
-	RequestProgress struct {
-		Enabled bool
-	}
-}
-
 // ObjectIdentifier carries key name for the object to delete.
 type ObjectIdentifier struct {
 	ObjectName string `xml:"Key"`

cmd/api-errors.go

@@ -187,11 +187,15 @@ const (
 	// new error codes here.
 
 	ErrMalformedJSON
+	ErrAdminNoSuchUser
+	ErrAdminNoSuchPolicy
+	ErrAdminInvalidArgument
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
 	ErrAdminConfigNoQuorum
 	ErrAdminConfigTooLarge
 	ErrAdminConfigBadJSON
+	ErrAdminConfigDuplicateKeys
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
@@ -249,7 +253,6 @@ const (
 	ErrParseUnsupportedAlias
 	ErrParseUnsupportedSyntax
 	ErrParseUnknownOperator
-	ErrParseInvalidPathComponent
 	ErrParseMissingIdentAfterAt
 	ErrParseUnexpectedOperator
 	ErrParseUnexpectedTerm
@@ -288,7 +291,6 @@ const (
 	ErrEvaluatorInvalidTimestampFormatPatternToken
 	ErrEvaluatorInvalidTimestampFormatPatternSymbol
 	ErrEvaluatorBindingDoesNotExist
-	ErrInvalidColumnIndex
 	ErrMissingHeaders
 	ErrAdminConfigNotificationTargetsFailed
 	ErrAdminProfilerNotEnabled
@@ -863,6 +865,21 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The JSON you provided was not well-formed or did not validate against our published format.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminNoSuchUser: {
+		Code:           "XMinioAdminNoSuchUser",
+		Description:    "The specified user does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminNoSuchPolicy: {
+		Code:           "XMinioAdminNoSuchPolicy",
+		Description:    "The canned policy does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminInvalidArgument: {
+		Code:           "XMinioAdminInvalidArgument",
+		Description:    "Invalid arguments specified.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminInvalidAccessKey: {
 		Code:           "XMinioAdminInvalidAccessKey",
 		Description:    "The access key is invalid.",
@@ -881,11 +898,16 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	ErrAdminConfigTooLarge: {
 		Code: "XMinioAdminConfigTooLarge",
 		Description: fmt.Sprintf("Configuration data provided exceeds the allowed maximum of %d bytes",
-			maxConfigJSONSize),
+			maxEConfigJSONSize),
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrAdminConfigBadJSON: {
 		Code:           "XMinioAdminConfigBadJSON",
+		Description:    "JSON configuration provided is of incorrect format",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminConfigDuplicateKeys: {
+		Code:           "XMinioAdminConfigDuplicateKeys",
 		Description:    "JSON configuration provided has objects with duplicate keys",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
@@ -1199,11 +1221,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The SQL expression contains an invalid operator.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrParseInvalidPathComponent: {
-		Code:           "ParseInvalidPathComponent",
-		Description:    "The SQL expression contains an invalid path component.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrParseMissingIdentAfterAt: {
 		Code:           "ParseMissingIdentAfterAt",
 		Description:    "Did not find the expected identifier after the @ symbol in the SQL expression.",
@@ -1389,11 +1406,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Time stamp format pattern contains an invalid symbol in the SQL expression.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrInvalidColumnIndex: {
-		Code:           "InvalidColumnIndex",
-		Description:    "Column index in the SQL expression is invalid.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrEvaluatorBindingDoesNotExist: {
 		Code:           "ErrEvaluatorBindingDoesNotExist",
 		Description:    "A column name or a path provided does not exist in the SQL expression",
@@ -1422,6 +1434,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 	// Verify if the underlying error is signature mismatch.
 	switch err {
+	case errInvalidArgument:
+		apiErr = ErrAdminInvalidArgument
+	case errNoSuchUser:
+		apiErr = ErrAdminNoSuchUser
+	case errNoSuchPolicy:
+		apiErr = ErrAdminNoSuchPolicy
 	case errSignatureMismatch:
 		apiErr = ErrSignatureDoesNotMatch
 	case errInvalidRange:
@@ -1435,10 +1453,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
 	// SSE errors
+	case errInvalidEncryptionParameters:
+		apiErr = ErrInvalidEncryptionParameters
 	case crypto.ErrInvalidEncryptionMethod:
 		apiErr = ErrInvalidEncryptionMethod
-	case errInsecureSSERequest:
-		apiErr = ErrInsecureSSECustomerRequest
 	case crypto.ErrInvalidCustomerAlgorithm:
 		apiErr = ErrInvalidSSECustomerAlgorithm
 	case crypto.ErrInvalidCustomerKey:
@@ -1547,8 +1565,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrParseUnsupportedSyntax
 	case s3select.ErrParseUnknownOperator:
 		apiErr = ErrParseUnknownOperator
-	case s3select.ErrParseInvalidPathComponent:
-		apiErr = ErrParseInvalidPathComponent
 	case s3select.ErrParseMissingIdentAfterAt:
 		apiErr = ErrParseMissingIdentAfterAt
 	case s3select.ErrParseUnexpectedOperator:
@@ -1621,8 +1637,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEvaluatorInvalidTimestampFormatPatternToken
 	case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbol:
 		apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbol
-	case s3select.ErrInvalidColumnIndex:
-		apiErr = ErrInvalidColumnIndex
 	case s3select.ErrEvaluatorBindingDoesNotExist:
 		apiErr = ErrEvaluatorBindingDoesNotExist
 	case s3select.ErrMissingHeaders:

cmd/api-errors_test.go

@@ -52,7 +52,6 @@ var toAPIErrorCodeTests = []struct {
 	{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
 
 	// SSE-C errors
-	{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
 	{err: crypto.ErrInvalidCustomerAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
 	{err: crypto.ErrMissingCustomerKey, errCode: ErrMissingSSECustomerKey},
 	{err: crypto.ErrInvalidCustomerKey, errCode: ErrInvalidSSECustomerKey},

cmd/api-resources.go

@@ -22,15 +22,22 @@ import (
 )
 
 // Parse bucket url queries
-func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
-	prefix = values.Get("prefix")
-	marker = values.Get("marker")
-	delimiter = values.Get("delimiter")
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
 	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
 	} else {
 		maxkeys = maxObjectList
 	}
+
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	return
 }
@@ -47,44 +54,69 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 		}
 	}
 
+	if values.Get("max-keys") != "" {
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
+	} else {
+		maxkeys = maxObjectList
+	}
+
 	prefix = values.Get("prefix")
 	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
-	} else {
-		maxkeys = maxObjectList
-	}
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse bucket url queries for ?uploads
-func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	if values.Get("max-uploads") != "" {
+		var err error
+		if maxUploads, err = strconv.Atoi(values.Get("max-uploads")); err != nil {
+			errCode = ErrInvalidMaxUploads
+			return
+		}
+	} else {
+		maxUploads = maxUploadsList
+	}
+
 	prefix = values.Get("prefix")
 	keyMarker = values.Get("key-marker")
 	uploadIDMarker = values.Get("upload-id-marker")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-uploads") != "" {
-		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
-	} else {
-		maxUploads = maxUploadsList
-	}
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse object url queries
-func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
-	uploadID = values.Get("uploadId")
-	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string, errCode APIErrorCode) {
+	var err error
+	errCode = ErrNone
+
 	if values.Get("max-parts") != "" {
-		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+		if maxParts, err = strconv.Atoi(values.Get("max-parts")); err != nil {
+			errCode = ErrInvalidMaxParts
+			return
+		}
 	} else {
 		maxParts = maxPartsList
 	}
+
+	if values.Get("part-number-marker") != "" {
+		if partNumberMarker, err = strconv.Atoi(values.Get("part-number-marker")); err != nil {
+			errCode = ErrInvalidPartNumberMarker
+			return
+		}
+	}
+
+	uploadID = values.Get("uploadId")
 	encodingType = values.Get("encoding-type")
 	return
 }

cmd/api-resources_test.go

@@ -156,7 +156,10 @@ func TestListObjectsV1Resources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		prefix, marker, delimiter, maxKeys, encodingType := getListObjectsV1Args(testCase.values)
+		prefix, marker, delimiter, maxKeys, encodingType, argsErr := getListObjectsV1Args(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}
@@ -198,7 +201,10 @@ func TestGetObjectsResources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		uploadID, partNumberMarker, maxParts, encodingType := getObjectResources(testCase.values)
+		uploadID, partNumberMarker, maxParts, encodingType, argsErr := getObjectResources(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if uploadID != testCase.uploadID {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.uploadID, uploadID)
 		}

cmd/auth-handler.go

@@ -27,8 +27,10 @@ import (
 	"net/http"
 	"strings"
 
+	jwtgo "github.com/dgrijalva/jwt-go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -86,6 +88,7 @@ const (
 	authTypeSigned
 	authTypeSignedV2
 	authTypeJWT
+	authTypeSTS
 )
 
 // Get request authentication type.
@@ -106,6 +109,8 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypePostPolicy
 	} else if _, ok := r.Header["Authorization"]; !ok {
 		return authTypeAnonymous
+	} else if _, ok := r.URL.Query()["Action"]; ok {
+		return authTypeSTS
 	}
 	return authTypeUnknown
 }
@@ -114,7 +119,21 @@ func getRequestAuthType(r *http.Request) authType {
 // It does not accept presigned or JWT or anonymous requests.
 func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
-	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok && getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) { // we only support V4 (no presign) with auth. body
+	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
+		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
+		// We only support admin credentials to access admin APIs.
+
+		var owner bool
+		_, owner, s3Err = getReqAccessKeyV4(r, region)
+		if s3Err != ErrNone {
+			return s3Err
+		}
+
+		if !owner {
+			return ErrAccessDenied
+		}
+
+		// we only support V4 (no presign) with auth body
 		s3Err = isReqAuthenticated(r, region)
 	}
 	if s3Err != ErrNone {
@@ -125,30 +144,62 @@ func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 	return s3Err
 }
 
-func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) APIErrorCode {
-	isOwner := true
-	accountName := globalServerConfig.GetCredential().AccessKey
+// Fetch the security token set by the client.
+func getSessionToken(r *http.Request) (token string) {
+	token = r.Header.Get("X-Amz-Security-Token")
+	if token != "" {
+		return token
+	}
+	return r.URL.Query().Get("X-Amz-Security-Token")
+}
 
+// Fetch claims in the security token returned by the client and validate the token.
+func getClaimsFromToken(r *http.Request) (map[string]interface{}, APIErrorCode) {
+	claims := make(map[string]interface{})
+	token := getSessionToken(r)
+	if token == "" {
+		return nil, ErrNone
+	}
+	p := &jwtgo.Parser{}
+	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
+	if err != nil {
+		return nil, toAPIErrorCode(errAuthentication)
+	}
+	if !jtoken.Valid {
+		return nil, toAPIErrorCode(errAuthentication)
+	}
+	return claims, ErrNone
+}
+
+// Check request auth type verifies the incoming http request
+// - validates the request signature
+// - validates the policy action if anonymous tests bucket policies if any,
+//   for authenticated requests validates IAM policies.
+// returns APIErrorCode if any to be replied to the client.
+func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
+	var accessKey string
+	var owner bool
 	switch getRequestAuthType(r) {
-	case authTypeUnknown:
+	case authTypeUnknown, authTypeStreamingSigned:
 		return ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
-		if errorCode := isReqAuthenticatedV2(r); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
+			return s3Err
 		}
+		accessKey, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
 		region := globalServerConfig.GetRegion()
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
 		}
-
-		if errorCode := isReqAuthenticated(r, region); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticated(r, region); s3Err != ErrNone {
+			return s3Err
 		}
-	default:
-		isOwner = false
-		accountName = ""
+		accessKey, owner, s3Err = getReqAccessKeyV4(r, region)
+	}
+	if s3Err != ErrNone {
+		return s3Err
 	}
 
 	// LocationConstraint is valid only for CreateBucketAction.
@@ -174,17 +225,36 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
 
-	if globalPolicySys.IsAllowed(policy.Args{
-		AccountName:     accountName,
-		Action:          action,
+	claims, s3Err := getClaimsFromToken(r)
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	if accessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     accessKey,
+			Action:          action,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, locationConstraint),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     accessKey,
+		Action:          iampolicy.Action(action),
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, locationConstraint),
-		IsOwner:         isOwner,
+		ConditionValues: getConditionValues(r, ""),
 		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
 	}) {
 		return ErrNone
 	}
-
 	return ErrAccessDenied
 }
 
@@ -288,7 +358,7 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	} else if aType == authTypeJWT {
 		// Validate Authorization header if its valid for JWT request.
-		if !isHTTPRequestValid(r) {
+		if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
 			w.WriteHeader(http.StatusUnauthorized)
 			return
 		}
@@ -297,3 +367,55 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
 }
+
+// isPutAllowed - check if PUT operation is allowed on the resource, this
+// call verifies bucket policies and IAM policies, supports multi user
+// checks etc.
+func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request) (s3Err APIErrorCode) {
+	var accessKey string
+	var owner bool
+	switch atype {
+	case authTypeUnknown:
+		return ErrAccessDenied
+	case authTypeSignedV2, authTypePresignedV2:
+		accessKey, owner, s3Err = getReqAccessKeyV2(r)
+	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
+		region := globalServerConfig.GetRegion()
+		accessKey, owner, s3Err = getReqAccessKeyV4(r, region)
+	}
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	claims, s3Err := getClaimsFromToken(r)
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	if accessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     accessKey,
+			Action:          policy.PutObjectAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     accessKey,
+		Action:          policy.PutObjectAction,
+		BucketName:      bucketName,
+		ConditionValues: getConditionValues(r, ""),
+		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
+	}) {
+		return ErrNone
+	}
+	return ErrAccessDenied
+}

cmd/bucket-handlers-listobjects.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/policy"
 )
@@ -58,6 +59,8 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -137,6 +140,8 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -152,7 +157,11 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	// Extract all the litsObjectsV1 query params to their native values.
-	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
+	prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 
 	// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
 	// does not throw an error.

cmd/bucket-handlers.go

@@ -89,6 +89,8 @@ func initFederatorBackend(objLayer ObjectLayer) {
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLocation")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -137,6 +139,8 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -151,7 +155,11 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 	if maxUploads < 0 {
 		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
 		return
@@ -184,6 +192,8 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBuckets")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -384,6 +394,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucket")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -460,6 +472,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PostPolicyBucket")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -671,6 +685,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	default:
 		writeSuccessNoContent(w)
 	}
+
+	for k, v := range objInfo.UserDefined {
+		logger.GetReqInfo(ctx).SetTags(k, v)
+	}
+
+	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // HeadBucketHandler - HEAD Bucket
@@ -682,6 +702,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadBucket")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -712,6 +734,8 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucket")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 

cmd/bucket-notification-handlers.go

@@ -44,6 +44,8 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketNotification")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
@@ -96,6 +98,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketNotification")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -139,7 +143,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err = saveNotificationConfig(objectAPI, bucketName, config); err != nil {
+	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -156,6 +160,8 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListenBucketNotification")
 
+	defer logger.AuditLog(ctx, r)
+
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-policy-handlers.go

@@ -40,6 +40,8 @@ const (
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketPolicy")
 
+	defer logger.AuditLog(ctx, r)
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -101,6 +103,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketPolicy")
 
+	defer logger.AuditLog(ctx, r)
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -137,6 +141,8 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketPolicy")
 
+	defer logger.AuditLog(ctx, r)
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)

cmd/common-main.go

@@ -28,12 +28,11 @@ import (
 	etcd "github.com/coreos/etcd/clientv3"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
-
-	"github.com/minio/minio-go/pkg/set"
 )
 
 // Check for updates and print a notification message
@@ -50,10 +49,17 @@ func checkUpdate(mode string) {
 
 // Load logger targets based on user's configuration
 func loadLoggers() {
+	if endpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT"); ok {
+		// Enable http logging through ENV, this is specifically added gateway audit logging.
+		logger.AddTarget(logger.NewHTTP(endpoint, NewCustomHTTPTransport()))
+		return
+	}
+
 	if globalServerConfig.Logger.Console.Enabled {
 		// Enable console logging
 		logger.AddTarget(logger.NewConsole())
 	}
+
 	for _, l := range globalServerConfig.Logger.HTTP {
 		if l.Enabled {
 			// Enable http logging
@@ -122,6 +128,7 @@ func handleCommonEnvVars() {
 		if err != nil {
 			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
 		}
+		cred.Expiration = timeSentinel
 
 		// credential Envs are set globally.
 		globalIsEnvCreds = true
@@ -131,7 +138,7 @@ func handleCommonEnvVars() {
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
 		browserFlag, err := ParseBoolFlag(browser)
 		if err != nil {
-			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER environment variable")
+			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -162,7 +169,7 @@ func handleCommonEnvVars() {
 	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
 	if globalDomainName != "" {
 		if _, ok = dns2.IsDomainName(globalDomainName); !ok {
-			logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN environment variable")
+			logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN value in environment variable")
 		}
 	}
 
@@ -257,7 +264,7 @@ func handleCommonEnvVars() {
 	if worm := os.Getenv("MINIO_WORM"); worm != "" {
 		wormFlag, err := ParseBoolFlag(worm)
 		if err != nil {
-			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Unable to validate MINIO_WORM environment variable")
+			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
 		}
 
 		// worm Envs are set globally, this does not represent

cmd/config-common.go

@@ -0,0 +1,133 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+)
+
+var errConfigNotFound = errors.New("config file not found")
+
+func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+	var buffer bytes.Buffer
+	// Read entire content by setting size to -1
+	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return nil, errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return nil, err
+	}
+
+	// Return config not found on empty content.
+	if buffer.Len() == 0 {
+		return nil, errConfigNotFound
+	}
+
+	return buffer.Bytes(), nil
+}
+
+func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	_, err := client.Delete(ctx, configFile)
+	return err
+}
+
+func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+}
+
+func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
+	_, err := client.Put(ctx, configFile, string(data))
+	return err
+}
+
+func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
+	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
+	if err != nil {
+		return err
+	}
+
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, hashReader, nil, ObjectOptions{})
+	return err
+}
+
+func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
+	resp, err := client.Get(ctx, configFile)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Count == 0 {
+		return nil, errConfigNotFound
+	}
+	for _, ev := range resp.Kvs {
+		if string(ev.Key) == configFile {
+			return ev.Value, nil
+		}
+	}
+	return nil, errConfigNotFound
+}
+
+// watchConfig - watches for changes on `configFile` on etcd and loads them.
+func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
+	if globalEtcdClient != nil {
+		for watchResp := range globalEtcdClient.Watch(context.Background(), configFile) {
+			for _, event := range watchResp.Events {
+				if event.IsModify() || event.IsCreate() {
+					loadCfgFn(objAPI)
+				}
+			}
+		}
+	}
+}
+
+func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	resp, err := globalEtcdClient.Get(ctx, configFile)
+	if err != nil {
+		return err
+	}
+	if resp.Count == 0 {
+		return errConfigNotFound
+	}
+	return nil
+}
+
+func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	if globalEtcdClient != nil {
+		return checkConfigEtcd(ctx, globalEtcdClient, configFile)
+	}
+
+	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return err
+	}
+	return nil
+}

cmd/config-current.go

@@ -20,15 +20,18 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"sync"
 
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
-
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Steps to move from version N to version N+1
@@ -40,9 +43,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "30"
+const serverConfigVersion = "31"
 
-type serverConfig = serverConfigV30
+type serverConfig = serverConfigV31
 
 var (
 	// globalServerConfig server config.
@@ -173,55 +176,55 @@ func (s *serverConfig) Validate() error {
 	// Worm, Cache and StorageClass values are already validated during json unmarshal
 	for _, v := range s.Notify.AMQP {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("amqp: %s", err.Error())
+			return fmt.Errorf("amqp: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Elasticsearch {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("elasticsearch: %s", err.Error())
+			return fmt.Errorf("elasticsearch: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Kafka {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("kafka: %s", err.Error())
+			return fmt.Errorf("kafka: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.MQTT {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("mqtt: %s", err.Error())
+			return fmt.Errorf("mqtt: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.MySQL {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("mysql: %s", err.Error())
+			return fmt.Errorf("mysql: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.NATS {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("nats: %s", err.Error())
+			return fmt.Errorf("nats: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.PostgreSQL {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("postgreSQL: %s", err.Error())
+			return fmt.Errorf("postgreSQL: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Redis {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("redis: %s", err.Error())
+			return fmt.Errorf("redis: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Webhook {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("webhook: %s", err.Error())
+			return fmt.Errorf("webhook: %s", err)
 		}
 	}
 
@@ -269,6 +272,20 @@ func (s *serverConfig) loadFromEnvs() {
 	if globalIsEnvCompression {
 		s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
 	}
+
+	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
+		if u, err := xnet.ParseURL(jwksURL); err == nil {
+			s.OpenID.JWKS.URL = u
+			s.OpenID.JWKS.PopulatePublicKey()
+		}
+	}
+
+	if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
+		if u, err := xnet.ParseURL(opaURL); err == nil {
+			s.Policy.OPA.URL = u
+			s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN")
+		}
+	}
 }
 
 // TestNotificationTargets tries to establish connections to all notification
@@ -503,17 +520,31 @@ func (s *serverConfig) loadToCachedConfigs() {
 			globalKMSKeyID = globalKMSConfig.Vault.Key.Name
 		}
 	}
+
 	if !globalIsCompressionEnabled {
 		compressionConf := s.GetCompressionConfig()
 		globalCompressExtensions = compressionConf.Extensions
 		globalCompressMimeTypes = compressionConf.MimeTypes
 		globalIsCompressionEnabled = compressionConf.Enabled
 	}
+
+	if globalIAMValidators == nil {
+		globalIAMValidators = getAuthValidators(s)
+	}
+
+	if globalPolicyOPA == nil {
+		if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
+			globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
+				URL:       s.Policy.OPA.URL,
+				AuthToken: s.Policy.OPA.AuthToken,
+			})
+		}
+	}
 }
 
-// newConfig - initialize a new server config, saves env parameters if
+// newSrvConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
-func newConfig(objAPI ObjectLayer) error {
+func newSrvConfig(objAPI ObjectLayer) error {
 	// Initialize server config.
 	srvCfg := newServerConfig()
 
@@ -564,6 +595,20 @@ func loadConfig(objAPI ObjectLayer) error {
 	return nil
 }
 
+// getAuthValidators - returns ValidatorList which contains
+// enabled providers in server config.
+// A new authentication provider is added like below
+// * Add a new provider in pkg/iam/validator package.
+func getAuthValidators(config *serverConfig) *validator.Validators {
+	validators := validator.NewValidators()
+
+	if config.OpenID.JWKS.URL != nil {
+		validators.Add(validator.NewJWT(config.OpenID.JWKS))
+	}
+
+	return validators
+}
+
 // getNotificationTargets - returns TargetList which contains enabled targets in serverConfig.
 // A new notification target is added like below
 // * Add a new target in pkg/event/target package.

cmd/config-current_test.go

@@ -237,7 +237,7 @@ func TestValidateConfig(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		if err = saveConfig(objLayer, configPath, []byte(testCase.configData)); err != nil {
+		if err = saveConfig(context.Background(), objLayer, configPath, []byte(testCase.configData)); err != nil {
 			t.Fatal(err)
 		}
 		_, err = getValidConfig(objLayer)
@@ -260,8 +260,16 @@ func TestConfigDiff(t *testing.T) {
 		{&serverConfig{}, nil, "Given configuration is empty"},
 		// 2
 		{
-			&serverConfig{Credential: auth.Credentials{"u1", "p1"}},
-			&serverConfig{Credential: auth.Credentials{"u1", "p2"}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p1",
+				Expiration: timeSentinel,
+			}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p2",
+				Expiration: timeSentinel,
+			}},
 			"Credential configuration differs",
 		},
 		// 3

cmd/config-migrate.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path"
@@ -29,6 +30,8 @@ import (
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
 )
@@ -2410,7 +2413,7 @@ func migrateV27ToV28() error {
 	return nil
 }
 
-// Migrates '.minio.sys/config.json' to v30.
+// Migrates '.minio.sys/config.json' to v31.
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
 		return err
@@ -2418,73 +2421,167 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	if err := migrateV28ToV29MinioSys(objAPI); err != nil {
 		return err
 	}
-	return migrateV29ToV30MinioSys(objAPI)
+	if err := migrateV29ToV30MinioSys(objAPI); err != nil {
+		return err
+	}
+	return migrateV30ToV31MinioSys(objAPI)
 }
 
-func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
+func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
+	data, err := readConfig(context.Background(), objAPI, configFile)
+	if err != nil {
+		return false, nil, err
+	}
+
+	var versionConfig struct {
+		Version string `json:"version"`
+	}
+
+	vcfg := &versionConfig
+	if err = json.Unmarshal(data, vcfg); err != nil {
+		return false, nil, err
+	}
+	return vcfg.Version == version, data, nil
+}
+
+func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	srvConfig, err := readServerConfig(context.Background(), objAPI)
+	ok, data, err := checkConfigVersion(objAPI, configFile, "27")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
 	}
-	if srvConfig.Version != "29" {
+	if !ok {
 		return nil
 	}
 
-	srvConfig.Version = "30"
-	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
-	srvConfig.Compression.Enabled = false
-	srvConfig.Compression.Extensions = globalCompressExtensions
-	srvConfig.Compression.MimeTypes = globalCompressMimeTypes
-	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from  29  to  30 . %v", err)
+	cfg := &serverConfigV28{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
 	}
 
-	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
+	cfg.Version = "28"
+	cfg.KMS = crypto.KMSConfig{}
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
 	return nil
 }
 
 func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	srvConfig, err := readServerConfig(context.Background(), objAPI)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "28")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
 	}
-	if srvConfig.Version != "28" {
+	if !ok {
 		return nil
 	}
 
-	srvConfig.Version = "29"
-	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
+	cfg := &serverConfigV29{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	cfg.Version = "29"
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "28", "29")
 	return nil
 }
 
-func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
+func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	srvConfig, err := readServerConfig(context.Background(), objAPI)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "29")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
 	}
-	if srvConfig.Version != "27" {
+	if !ok {
 		return nil
 	}
 
-	srvConfig.Version = "28"
-	srvConfig.KMS = crypto.KMSConfig{}
-	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
+	cfg := &serverConfigV30{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
 	}
 
-	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
+	cfg.Version = "30"
+	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
+	cfg.Compression.Enabled = false
+	cfg.Compression.Extensions = globalCompressExtensions
+	cfg.Compression.MimeTypes = globalCompressMimeTypes
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
+	return nil
+}
+
+func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "30")
+	if err == errConfigNotFound {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+	if !ok {
+		return nil
+	}
+
+	cfg := &serverConfigV31{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	cfg.Version = "31"
+	cfg.OpenID.JWKS = validator.JWKSArgs{
+		URL: &xnet.URL{},
+	}
+	cfg.Policy.OPA = iampolicy.OpaArgs{
+		URL:       &xnet.URL{},
+		AuthToken: "",
+	}
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "30", "31")
 	return nil
 }

cmd/config-migrate_test.go

@@ -159,8 +159,8 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	}
 }
 
-// Test if a config migration from v2 to v29 is successfully done
-func TestServerConfigMigrateV2toV29(t *testing.T) {
+// Test if a config migration from v2 to v30 is successfully done
+func TestServerConfigMigrateV2toV30(t *testing.T) {
 	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
 	if err != nil {
 		t.Fatal(err)
@@ -222,6 +222,7 @@ func TestServerConfigMigrateV2toV29(t *testing.T) {
 	if globalServerConfig.Credential.AccessKey != accessKey {
 		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, globalServerConfig.Credential.AccessKey)
 	}
+
 	if globalServerConfig.Credential.SecretKey != secretKey {
 		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, globalServerConfig.Credential.SecretKey)
 	}

cmd/config-versions.go

@@ -22,6 +22,8 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -755,6 +757,9 @@ type serverConfigV28 struct {
 	Logger loggerConfig `json:"logger"`
 }
 
+// serverConfigV29 is just like version '28'.
+type serverConfigV29 serverConfigV28
+
 // compressionConfig represents the compression settings.
 type compressionConfig struct {
 	Enabled    bool     `json:"enabled"`
@@ -765,8 +770,6 @@ type compressionConfig struct {
 // serverConfigV30 is just like version '29', stores additionally
 // extensions and mimetypes fields for compression.
 type serverConfigV30 struct {
-	quick.Config `json:"-"` // ignore interfaces
-
 	Version string `json:"version"`
 
 	// S3 API configuration.
@@ -792,3 +795,45 @@ type serverConfigV30 struct {
 	// Compression configuration
 	Compression compressionConfig `json:"compress"`
 }
+
+// serverConfigV31 is just like version '30', with OPA and OpenID configuration.
+type serverConfigV31 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+
+	// Compression configuration
+	Compression compressionConfig `json:"compress"`
+
+	// OpenID configuration
+	OpenID struct {
+		// JWKS validator config.
+		JWKS validator.JWKSArgs `json:"jwks"`
+	} `json:"openid"`
+
+	// External policy enforcements.
+	Policy struct {
+		// OPA configuration.
+		OPA iampolicy.OpaArgs `json:"opa"`
+
+		// Add new external policy enforcements here.
+	} `json:"policy"`
+}

cmd/config.go

@@ -20,16 +20,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
-	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -62,16 +59,10 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 	}
 
 	// Create a backup of the current config
-	reader, err := readConfig(ctx, objAPI, configFile)
+	oldData, err := readConfig(ctx, objAPI, configFile)
 	if err == nil {
-		var oldData []byte
-		oldData, err = ioutil.ReadAll(reader)
-		if err != nil {
-			return err
-		}
 		backupConfigFile := path.Join(minioConfigPrefix, minioConfigBackupFile)
-		err = saveConfig(objAPI, backupConfigFile, oldData)
-		if err != nil {
+		if err = saveConfig(ctx, objAPI, backupConfigFile, oldData); err != nil {
 			return err
 		}
 	} else {
@@ -81,40 +72,18 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 	}
 
 	// Save the new config in the std config path
-	return saveConfig(objAPI, configFile, data)
-}
-
-func readConfigEtcd(configFile string) ([]byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	resp, err := globalEtcdClient.Get(ctx, configFile)
-	defer cancel()
-	if err != nil {
-		return nil, err
-	}
-	if resp.Count == 0 {
-		return nil, errConfigNotFound
-	}
-	for _, ev := range resp.Kvs {
-		if string(ev.Key) == configFile {
-			return ev.Value, nil
-		}
-	}
-	return nil, errConfigNotFound
+	return saveConfig(ctx, objAPI, configFile, data)
 }
 
 func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
 	var configData []byte
 	var err error
+
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	if globalEtcdClient != nil {
-		configData, err = readConfigEtcd(configFile)
+		configData, err = readConfigEtcd(ctx, globalEtcdClient, configFile)
 	} else {
-		var reader io.Reader
-		reader, err = readConfig(ctx, objAPI, configFile)
-		if err != nil {
-			return nil, err
-		}
-		configData, err = ioutil.ReadAll(reader)
+		configData, err = readConfig(ctx, objAPI, configFile)
 	}
 	if err != nil {
 		return nil, err
@@ -129,82 +98,17 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, e
 	}
 
 	var config = &serverConfig{}
-	if err := json.Unmarshal(configData, config); err != nil {
+	if err = json.Unmarshal(configData, config); err != nil {
 		return nil, err
 	}
 
-	if err := quick.CheckData(config); err != nil {
+	if err = quick.CheckData(config); err != nil {
 		return nil, err
 	}
 
 	return config, nil
 }
 
-func checkServerConfigEtcd(configFile string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	resp, err := globalEtcdClient.Get(ctx, configFile)
-	defer cancel()
-	if err != nil {
-		return err
-	}
-	if resp.Count == 0 {
-		return errConfigNotFound
-	}
-	return nil
-}
-
-func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	if globalEtcdClient != nil {
-		return checkServerConfigEtcd(configFile)
-	}
-
-	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
-		// Convert ObjectNotFound to errConfigNotFound
-		if isErrObjectNotFound(err) {
-			return errConfigNotFound
-		}
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
-		return err
-	}
-	return nil
-}
-
-func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
-	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
-	if err != nil {
-		return err
-	}
-
-	_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil, ObjectOptions{})
-	return err
-}
-
-var errConfigNotFound = errors.New("config file not found")
-
-func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
-	var buffer bytes.Buffer
-	// Read entire content by setting size to -1
-	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
-		// Convert ObjectNotFound and IncompleteBody errors into errConfigNotFound
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, errConfigNotFound
-		}
-
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
-		return nil, err
-	}
-
-	// Return config not found on empty content.
-	if buffer.Len() == 0 {
-		return nil, errConfigNotFound
-	}
-
-	return &buffer, nil
-}
-
 // ConfigSys - config system.
 type ConfigSys struct{}
 
@@ -234,7 +138,8 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 		case _ = <-retryTimerCh:
 			err := initConfig(objAPI)
 			if err != nil {
-				if isInsufficientReadQuorum(err) || isInsufficientWriteQuorum(err) {
+				if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
 					logger.Info("Waiting for configuration to be initialized..")
 					continue
 				}
@@ -255,8 +160,9 @@ func NewConfigSys() *ConfigSys {
 func migrateConfigToMinioSys(objAPI ObjectLayer) error {
 	defer os.Rename(getConfigFile(), getConfigFile()+".deprecated")
 
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	// Verify if backend already has the file.
-	if err := checkServerConfig(context.Background(), objAPI); err != errConfigNotFound {
+	if err := checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
@@ -285,7 +191,14 @@ func initConfig(objAPI ObjectLayer) error {
 		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
 		cancel()
 		if err == nil && resp.Count > 0 {
-			return migrateConfig()
+			if err = migrateConfig(); err != nil {
+				return err
+			}
+
+			// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
+			if err := migrateConfigToMinioSys(objAPI); err != nil {
+				return err
+			}
 		}
 	} else {
 		if isFile(getConfigFile()) {
@@ -301,10 +214,15 @@ func initConfig(objAPI ObjectLayer) error {
 		}
 	}
 
-	if err := checkServerConfig(context.Background(), objAPI); err != nil {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	// Watch config for changes and reloads them in-memory.
+	go watchConfig(objAPI, configFile, loadConfig)
+
+	if err := checkConfig(context.Background(), objAPI, configFile); err != nil {
 		if err == errConfigNotFound {
 			// Config file does not exist, we create it fresh and return upon success.
-			if err = newConfig(objAPI); err != nil {
+			if err = newSrvConfig(objAPI); err != nil {
 				return err
 			}
 		} else {

cmd/crypto/key.go

@@ -140,3 +140,37 @@ func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
 	mac.Sum(partKey[:0])
 	return partKey
 }
+
+// SealETag seals the etag using the object key.
+// It does not encrypt empty ETags because such ETags indicate
+// that the S3 client hasn't sent an ETag = MD5(object) and
+// the backend can pick an ETag value.
+func (key ObjectKey) SealETag(etag []byte) []byte {
+	if len(etag) == 0 { // don't encrypt empty ETag - only if client sent ETag = MD5(object)
+		return etag
+	}
+	var buffer bytes.Buffer
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write([]byte("SSE-etag"))
+	if _, err := sio.Encrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to encrypt ETag using object key"))
+	}
+	return buffer.Bytes()
+}
+
+// UnsealETag unseals the etag using the provided object key.
+// It does not try to decrypt the ETag if len(etag) == 16
+// because such ETags indicate that the S3 client hasn't sent
+// an ETag = MD5(object) and the backend has picked an ETag value.
+func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
+	if !IsETagSealed(etag) {
+		return etag, nil
+	}
+	var buffer bytes.Buffer
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write([]byte("SSE-etag"))
+	if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}

cmd/crypto/key_test.go

@@ -166,3 +166,31 @@ func TestDerivePartKey(t *testing.T) {
 		}
 	}
 }
+
+var sealUnsealETagTests = []string{
+	"",
+	"90682b8e8cc7609c",
+	"90682b8e8cc7609c4671e1d64c73fc30",
+	"90682b8e8cc7609c4671e1d64c73fc307fb3104f",
+}
+
+func TestSealETag(t *testing.T) {
+	var key ObjectKey
+	for i := range key {
+		key[i] = byte(i)
+	}
+	for i, etag := range sealUnsealETagTests {
+		tag, err := hex.DecodeString(etag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decode etag: %s", i, err)
+		}
+		sealedETag := key.SealETag(tag)
+		unsealedETag, err := key.UnsealETag(sealedETag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decrypt etag: %s", i, err)
+		}
+		if !bytes.Equal(unsealedETag, tag) {
+			t.Errorf("Test %d: unsealed etag does not match: got %s - want %s", i, hex.EncodeToString(unsealedETag), etag)
+		}
+	}
+}

cmd/crypto/metadata.go

@@ -40,6 +40,18 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
 	delete(metadata, SSECopyKey)
 }
 
+// RemoveInternalEntries removes all crypto-specific internal
+// metadata entries from the metadata map.
+func RemoveInternalEntries(metadata map[string]string) {
+	delete(metadata, SSEMultipart)
+	delete(metadata, SSEIV)
+	delete(metadata, SSESealAlgorithm)
+	delete(metadata, SSECSealedKey)
+	delete(metadata, S3SealedKey)
+	delete(metadata, S3KMSKeyID)
+	delete(metadata, S3KMSSealedKey)
+}
+
 // IsEncrypted returns true if the object metadata indicates
 // that it was uploaded using some form of server-side-encryption.
 //
@@ -219,3 +231,6 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	copy(sealedKey.Key[:], encryptedKey)
 	return sealedKey, nil
 }
+
+// IsETagSealed returns true if the etag seems to be encrypted.
+func IsETagSealed(etag []byte) bool { return len(etag) > 16 }

cmd/crypto/metadata_test.go

@@ -17,6 +17,7 @@ package crypto
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/hex"
 	"testing"
 
 	"github.com/minio/minio/cmd/logger"
@@ -364,3 +365,75 @@ func TestSSECCreateMetadata(t *testing.T) {
 	}()
 	_ = SSEC.CreateMetadata(nil, SealedKey{Algorithm: InsecureSealAlgorithm})
 }
+
+var isETagSealedTests = []struct {
+	ETag     string
+	IsSealed bool
+}{
+	{ETag: "", IsSealed: false},                                                                                                // 0
+	{ETag: "90682b8e8cc7609c4671e1d64c73fc30", IsSealed: false},                                                                // 1
+	{ETag: "f201040c9dc593e39ea004dc1323699bcd", IsSealed: true},                                                               // 2 not valid ciphertext but looks like sealed ETag
+	{ETag: "20000f00fba2ee2ae4845f725964eeb9e092edfabc7ab9f9239e8344341f769a51ce99b4801b0699b92b16a72fa94972", IsSealed: true}, // 3
+}
+
+func TestIsETagSealed(t *testing.T) {
+	for i, test := range isETagSealedTests {
+		etag, err := hex.DecodeString(test.ETag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decode etag: %s", i, err)
+		}
+		if sealed := IsETagSealed(etag); sealed != test.IsSealed {
+			t.Errorf("Test %d: got %v - want %v", i, sealed, test.IsSealed)
+		}
+	}
+}
+
+var removeInternalEntriesTests = []struct {
+	Metadata, Expected map[string]string
+}{
+	{ // 0
+		Metadata: map[string]string{
+			SSEMultipart:     "",
+			SSEIV:            "",
+			SSESealAlgorithm: "",
+			SSECSealedKey:    "",
+			S3SealedKey:      "",
+			S3KMSKeyID:       "",
+			S3KMSSealedKey:   "",
+		},
+		Expected: map[string]string{},
+	},
+	{ // 1
+		Metadata: map[string]string{
+			SSEMultipart:         "",
+			SSEIV:                "",
+			"X-Amz-Meta-A":       "X",
+			"X-Minio-Internal-B": "Y",
+		},
+		Expected: map[string]string{
+			"X-Amz-Meta-A":       "X",
+			"X-Minio-Internal-B": "Y",
+		},
+	},
+}
+
+func TestRemoveInternalEntries(t *testing.T) {
+	isEqual := func(x, y map[string]string) bool {
+		if len(x) != len(y) {
+			return false
+		}
+		for k, v := range x {
+			if u, ok := y[k]; !ok || v != u {
+				return false
+			}
+		}
+		return true
+	}
+
+	for i, test := range removeInternalEntriesTests {
+		RemoveInternalEntries(test.Metadata)
+		if !isEqual(test.Metadata, test.Expected) {
+			t.Errorf("Test %d: got %v - want %v", i, test.Metadata, test.Expected)
+		}
+	}
+}

cmd/crypto/sse.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"errors"
 	"io"
+	"net/http"
+	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
@@ -69,10 +71,59 @@ const (
 // domain is "SSE-S3".
 func (s3) String() string { return "SSE-S3" }
 
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (sse s3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := sse.ParseMetadata(metadata)
+	if err != nil {
+		return
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return
+	}
+	err = key.Unseal(unsealKey, sealedKey, sse.String(), bucket, object)
+	return
+}
+
 // String returns the SSE domain as string. For SSE-C the
 // domain is "SSE-C".
 func (ssec) String() string { return "SSE-C" }
 
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-C client key of the HTTP headers
+// and returns the decrypted object key.
+func (sse ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := sse.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-Copy client key of the HTTP headers
+// and returns the decrypted object key.
+func (sse ssecCopy) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := sse.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// unsealObjectKey decrypts and returns the sealed object key
+// from the metadata using the SSE-C client key.
+func unsealObjectKey(clientKey [32]byte, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	sealedKey, err := SSEC.ParseMetadata(metadata)
+	if err != nil {
+		return
+	}
+	err = key.Unseal(clientKey, sealedKey, SSEC.String(), bucket, object)
+	return
+}
+
 // EncryptSinglePart encrypts an io.Reader which must be the
 // the body of a single-part PUT request.
 func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {

cmd/crypto/sse_test.go

@@ -14,7 +14,10 @@
 
 package crypto
 
-import "testing"
+import (
+	"net/http"
+	"testing"
+)
 
 func TestS3String(t *testing.T) {
 	const Domain = "SSE-S3"
@@ -29,3 +32,195 @@ func TestSSECString(t *testing.T) {
 		t.Errorf("SSEC's string method returns wrong domain: got '%s' - want '%s'", domain, Domain)
 	}
 }
+
+var ssecUnsealObjectKeyTests = []struct {
+	Headers        http.Header
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object2",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrSecretKeyMismatch,
+	},
+	{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":         "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+	{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5,
+	},
+}
+
+func TestSSECUnsealObjectKey(t *testing.T) {
+	for i, test := range ssecUnsealObjectKeyTests {
+		if _, err := SSEC.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}
+
+var sseCopyUnsealObjectKeyTests = []struct {
+	Headers        http.Header
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object2",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrSecretKeyMismatch,
+	},
+	{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":         "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+	{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5,
+	},
+}
+
+func TestSSECopyUnsealObjectKey(t *testing.T) {
+	for i, test := range sseCopyUnsealObjectKeyTests {
+		if _, err := SSECopy.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}
+
+var s3UnsealObjectKeyTests = []struct {
+	KMS            KMS
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid KMS key-ID and valid metadata entries for bucket/object
+		KMS:    NewKMS([32]byte{}),
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Iv":                "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":     "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":     "test-key-1",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":    "DAREv2-HMAC-SHA256",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid KMS key-ID for invalid metadata entries for bucket/object
+		KMS:    NewKMS([32]byte{}),
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Iv":                "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":     "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":     "test-key-1",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+}
+
+func TestS3UnsealObjectKey(t *testing.T) {
+	for i, test := range s3UnsealObjectKeyTests {
+		if _, err := S3.UnsealObjectKey(test.KMS, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}

cmd/disk-cache.go

@@ -38,8 +38,7 @@ import (
 )
 
 const (
-	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
-	cacheSizeMultiplier  = 100
+	// disk cache needs to have  object size space free for a cache entry to be created.
 	cacheTrashDir        = "trash"
 	cacheCleanupInterval = 10 // in minutes
 )
@@ -184,10 +183,7 @@ func (c cacheObjects) getMetadata(objInfo ObjectInfo) map[string]string {
 }
 
 func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-
-	objInfo, bkErr := c.GetObjectInfoFn(ctx, bucket, object, opts)
-
-	if c.isCacheExclude(bucket, object) || !objInfo.IsCacheable() {
+	if c.isCacheExclude(bucket, object) {
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
 
@@ -197,30 +193,29 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
 
-	backendDown := backendDownError(bkErr)
-	if bkErr != nil && !backendDown {
+	cacheReader, cacheErr := dcache.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+
+	objInfo, err := c.GetObjectInfoFn(ctx, bucket, object, opts)
+	if backendDownError(err) && cacheErr == nil {
+		return cacheReader, nil
+	} else if err != nil {
 		if _, ok := err.(ObjectNotFound); ok {
-			// Delete the cached entry if backend object was deleted.
+			// Delete cached entry if backend object was deleted.
 			dcache.Delete(ctx, bucket, object)
 		}
-		return nil, bkErr
+		return nil, err
 	}
 
-	if !backendDown && filterFromCache(objInfo.UserDefined) {
+	if !objInfo.IsCacheable() || filterFromCache(objInfo.UserDefined) {
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
 
-	if cacheReader, cacheErr := dcache.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts); cacheErr == nil {
-		if backendDown {
-			// If the backend is down, serve the request from cache.
-			return cacheReader, nil
-		}
-
+	if cacheErr == nil {
 		if cacheReader.ObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) {
 			// Object is not stale, so serve from cache
 			return cacheReader, nil
 		}
-
+		cacheReader.Close()
 		// Object is stale, so delete from cache
 		dcache.Delete(ctx, bucket, object)
 	}
@@ -232,8 +227,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		// We don't cache partial objects.
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
-	if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
-		// cache only objects < 1/100th of disk capacity
+	if !dcache.diskAvailable(objInfo.Size) {
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
 	}
 
@@ -290,6 +284,10 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		return err
 	}
 
+	if !backendDown && !objInfo.IsCacheable() {
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
+	}
+
 	if !backendDown && filterFromCache(objInfo.UserDefined) {
 		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
@@ -309,8 +307,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		// We don't cache partial objects.
 		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
-	if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
-		// cache only objects < 1/100th of disk capacity
+	if !dcache.diskAvailable(objInfo.Size) {
 		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// Initialize pipe.
@@ -657,7 +654,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 	size := r.Size()
 
 	// fetch from backend if there is no space on cache drive
-	if !dcache.diskAvailable(size * cacheSizeMultiplier) {
+	if !dcache.diskAvailable(size) {
 		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	// fetch from backend if cache exclude pattern or cache-control
@@ -749,9 +746,9 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
-	// make sure cache has at least cacheSizeMultiplier * size available
+	// make sure cache has at least size space available
 	size := data.Size()
-	if !dcache.diskAvailable(size * cacheSizeMultiplier) {
+	if !dcache.diskAvailable(size) {
 		select {
 		case dcache.purgeChan <- struct{}{}:
 		default:
@@ -969,6 +966,9 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
+		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+		},
 		PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts)
 		},

cmd/encryption-v1.go

@@ -37,7 +37,6 @@ import (
 
 var (
 	// AWS errors for invalid SSE-C requests.
-	errInsecureSSERequest   = errors.New("SSE-C requests require TLS connections")
 	errEncryptedObject      = errors.New("The object was stored using a form of SSE")
 	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
 	errKMSNotConfigured     = errors.New("KMS not configured for a server side encrypted object")
@@ -80,16 +79,31 @@ func hasServerSideEncryptionHeader(header http.Header) bool {
 	return crypto.S3.IsRequested(header) || crypto.SSEC.IsRequested(header)
 }
 
+// isEncryptedMultipart returns true if the current object is
+// uploaded by the user using multipart mechanism:
+// initiate new multipart, upload part, complete upload
+func isEncryptedMultipart(objInfo ObjectInfo) bool {
+	if len(objInfo.Parts) == 0 {
+		return false
+	}
+	if !crypto.IsMultiPart(objInfo.UserDefined) {
+		return false
+	}
+	for _, part := range objInfo.Parts {
+		_, err := sio.DecryptedSize(uint64(part.Size))
+		if err != nil {
+			return false
+		}
+	}
+	// Further check if this object is uploaded using multipart mechanism
+	// by the user and it is not about XL internally splitting the
+	// object into parts in PutObject()
+	return !(objInfo.backendType == BackendErasure && len(objInfo.ETag) == 32)
+}
+
 // ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.
 // It returns the client provided key on success.
 func ParseSSECopyCustomerRequest(h http.Header, metadata map[string]string) (key []byte, err error) {
-	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
-		// we cannot use r.TLS == nil here because Go's http implementation reflects on
-		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
-		// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
-		// will always fail -> r.TLS is always nil even for TLS requests.
-		return nil, errInsecureSSERequest
-	}
 	if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(h) {
 		return nil, crypto.ErrIncompatibleEncryptionMethod
 	}
@@ -106,13 +120,6 @@ func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {
 // ParseSSECustomerHeader parses the SSE-C header fields and returns
 // the client provided key on success.
 func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
-	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
-		// we cannot use r.TLS == nil here because Go's http implementation reflects on
-		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
-		// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
-		// will always fail -> r.TLS is always nil even for TLS requests.
-		return nil, errInsecureSSERequest
-	}
 	if crypto.S3.IsRequested(header) && crypto.SSEC.IsRequested(header) {
 		return key, crypto.ErrIncompatibleEncryptionMethod
 	}
@@ -361,7 +368,7 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
 // GetEncryptedOffsetLength - returns encrypted offset and length
 // along with sequence number
 func GetEncryptedOffsetLength(startOffset, length int64, objInfo ObjectInfo) (seqNumber uint32, encStartOffset, encLength int64) {
-	if len(objInfo.Parts) == 0 || !crypto.IsMultiPart(objInfo.UserDefined) {
+	if !isEncryptedMultipart(objInfo) {
 		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
 		return
 	}
@@ -378,7 +385,7 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
 	bucket, object := oi.Bucket, oi.Name
 
 	// Single part case
-	if len(oi.Parts) == 0 || !crypto.IsMultiPart(oi.UserDefined) {
+	if !isEncryptedMultipart(oi) {
 		var reader io.Reader
 		var err error
 		if copySource {
@@ -708,7 +715,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 	var seqNumber uint32
 	var encStartOffset, encLength int64
 
-	if len(objInfo.Parts) == 0 || !crypto.IsMultiPart(objInfo.UserDefined) {
+	if !isEncryptedMultipart(objInfo) {
 		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
 
 		var writer io.WriteCloser
@@ -870,7 +877,7 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if !crypto.IsEncrypted(o.UserDefined) {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
-	if len(o.Parts) == 0 || !crypto.IsMultiPart(o.UserDefined) {
+	if !isEncryptedMultipart(*o) {
 		size, err := sio.DecryptedSize(uint64(o.Size))
 		if err != nil {
 			err = errObjectTampered // assign correct error type
@@ -918,7 +925,7 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
 	}
 	sizes := []int64{int64(partSize)}
 	decObjSize = sizes[0]
-	if crypto.IsMultiPart(o.UserDefined) {
+	if isEncryptedMultipart(*o) {
 		sizes = make([]int64, len(o.Parts))
 		decObjSize = 0
 		for i, part := range o.Parts {

cmd/encryption-v1_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"encoding/base64"
 	"net/http"
 	"testing"
 
@@ -107,228 +106,6 @@ func TesthasSSECustomerHeader(t *testing.T) {
 	}
 }
 
-var parseSSECustomerRequestTests = []struct {
-	headers map[string]string
-	useTLS  bool
-	err     error
-}{
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: true, err: nil,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: false, err: errInsecureSSERequest,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES 256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: true, err: crypto.ErrInvalidCustomerAlgorithm,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
-			crypto.SSECKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
-			crypto.SSECKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
-			crypto.SSECKeyMD5:    " +jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
-			crypto.SSECKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		useTLS: true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "", // 7
-			crypto.SSECKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		useTLS: true, err: crypto.ErrMissingCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			crypto.SSECKeyMD5:    "",
-		},
-		useTLS: true, err: crypto.ErrMissingCustomerKeyMD5,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			crypto.SSECKeyMD5:    "",
-			crypto.SSEHeader:     "",
-		},
-		useTLS: true, err: crypto.ErrIncompatibleEncryptionMethod,
-	},
-}
-
-func TestParseSSECustomerRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	for i, test := range parseSSECustomerRequestTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		request := &http.Request{}
-		request.Header = headers
-		globalIsSSL = test.useTLS
-
-		_, err := ParseSSECustomerRequest(request)
-		if err != test.err {
-			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
-		}
-	}
-}
-
-var parseSSECopyCustomerRequestTests = []struct {
-	headers  map[string]string
-	metadata map[string]string
-	useTLS   bool
-	err      error
-}{
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: nil,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": base64.StdEncoding.EncodeToString(make([]byte, 64))},
-		useTLS:   true, err: crypto.ErrIncompatibleEncryptionMethod,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-		useTLS:   false, err: errInsecureSSERequest,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES 256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrInvalidCustomerAlgorithm,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
-			crypto.SSECopyKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
-			crypto.SSECopyKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
-			crypto.SSECopyKeyMD5:    " +jq/LwEOEO90YtiTuNFVw==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
-			crypto.SSECopyKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "", // 7
-			crypto.SSECopyKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrMissingCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			crypto.SSECopyKeyMD5:    "",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrMissingCustomerKeyMD5,
-	},
-}
-
-func TestParseSSECopyCustomerRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	for i, test := range parseSSECopyCustomerRequestTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		request := &http.Request{}
-		request.Header = headers
-		globalIsSSL = test.useTLS
-
-		_, err := ParseSSECopyCustomerRequest(request.Header, test.metadata)
-		if err != test.err {
-			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
-		}
-	}
-}
-
 var encryptRequestTests = []struct {
 	header   map[string]string
 	metadata map[string]string

cmd/erasure-utils.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"strings"
 
 	"github.com/klauspost/reedsolomon"
 	"github.com/minio/minio/cmd/logger"
@@ -82,7 +81,9 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		if write < int64(len(block)) {
 			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
 			if err != nil {
-				logger.LogIf(ctx, err)
+				if err != io.ErrClosedPipe {
+					logger.LogIf(ctx, err)
+				}
 				return 0, err
 			}
 			totalWritten += n
@@ -92,7 +93,7 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		n, err := io.Copy(dst, bytes.NewReader(block))
 		if err != nil {
 			// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
-			if !strings.Contains(err.Error(), "read/write on closed pipe") {
+			if err != io.ErrClosedPipe {
 				logger.LogIf(ctx, err)
 			}
 			return 0, err

cmd/fs-v1-multipart.go

@@ -713,6 +713,11 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 	}
 
 	fs.appendFileMapMu.Lock()
+	// Remove file in tmp folder
+	file := fs.appendFileMap[uploadID]
+	if file != nil {
+		fsRemoveFile(ctx, file.filePath)
+	}
 	delete(fs.appendFileMap, uploadID)
 	fs.appendFileMapMu.Unlock()
 
@@ -725,10 +730,14 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 		}
 		return toObjectErr(err, bucket, object)
 	}
+
 	// Ignore the error returned as Windows fails to remove directory if a file in it
 	// is Open()ed by the backgroundAppend()
 	fsRemoveAll(ctx, uploadIDDir)
 
+	// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
+	fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
+
 	return nil
 }
 

cmd/fs-v1.go

@@ -1018,7 +1018,7 @@ func (fs *FSObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
 	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
-		if err != nil {
+		if err != nil && err != errFileNotFound {
 			logger.LogIf(context.Background(), err)
 			return
 		}
@@ -1274,7 +1274,7 @@ func (fs *FSObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
 
 // SetBucketPolicy sets policy on bucket
 func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
-	return savePolicyConfig(fs, bucket, policy)
+	return savePolicyConfig(ctx, fs, bucket, policy)
 }
 
 // GetBucketPolicy will get policy on bucket

cmd/gateway-common.go

@@ -224,7 +224,16 @@ func FromMinioClientListBucketResultToV2Info(bucket string, result minio.ListBuc
 	}
 }
 
-// ToMinioClientMetadata converts metadata to map[string][]string
+// ToMinioClientObjectInfoMetadata convertes metadata to map[string][]string
+func ToMinioClientObjectInfoMetadata(metadata map[string]string) map[string][]string {
+	mm := make(map[string][]string, len(metadata))
+	for k, v := range metadata {
+		mm[http.CanonicalHeaderKey(k)] = []string{v}
+	}
+	return mm
+}
+
+// ToMinioClientMetadata converts metadata to map[string]string
 func ToMinioClientMetadata(metadata map[string]string) map[string]string {
 	mm := make(map[string]string)
 	for k, v := range metadata {
@@ -285,6 +294,8 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		err = BucketPolicyNotFound{}
 	case "InvalidBucketName":
 		err = BucketNameInvalid{Bucket: bucket}
+	case "InvalidPart":
+		err = InvalidPart{}
 	case "NoSuchBucket":
 		err = BucketNotFound{Bucket: bucket}
 	case "NoSuchKey":

cmd/gateway-main.go

@@ -157,28 +157,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Create certs path.
 	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
 
-	// Initialize server config.
-	srvCfg := newServerConfig()
-
-	// Override any values from ENVs.
-	srvCfg.loadFromEnvs()
-
-	// Load values to cached global values.
-	srvCfg.loadToCachedConfigs()
-
-	// hold the mutex lock before a new config is assigned.
-	globalServerConfigMu.Lock()
-	globalServerConfig = srvCfg
-	globalServerConfigMu.Unlock()
-
-	var cacheConfig = globalServerConfig.GetCacheConfig()
-	if len(cacheConfig.Drives) > 0 {
-		var err error
-		// initialize the new disk cache objects.
-		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
-		logger.FatalIf(err, "Unable to initialize disk caching")
-	}
-
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
@@ -189,14 +167,16 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	initNSLock(false) // Enable local namespace lock.
 
-	// Create new notification system.
-	globalNotificationSys = NewNotificationSys(globalServerConfig, EndpointList{})
-
-	// Create new policy system.
-	globalPolicySys = NewPolicySys()
-
 	router := mux.NewRouter().SkipClean(true)
 
+	if globalEtcdClient != nil {
+		// Enable STS router if etcd is enabled.
+		registerSTSRouter(router)
+
+		// Enable admin router if etcd is enabled.
+		registerAdminRouter(router)
+	}
+
 	// Add healthcheck router
 	registerHealthCheckRouter(router)
 
@@ -234,8 +214,59 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 
+	// Create a new config system.
+	globalConfigSys = NewConfigSys()
+
+	// Initialize server config.
+	srvCfg := newServerConfig()
+
+	// Override any values from ENVs.
+	srvCfg.loadFromEnvs()
+
+	// Load values to cached global values.
+	srvCfg.loadToCachedConfigs()
+
+	// hold the mutex lock before a new config is assigned.
+	globalServerConfigMu.Lock()
+	globalServerConfig = srvCfg
+	globalServerConfigMu.Unlock()
+
+	// Load logger subsystem
+	loadLoggers()
+
+	// This is only to uniquely identify each gateway deployments.
+	logger.SetDeploymentID(os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID"))
+
+	var cacheConfig = globalServerConfig.GetCacheConfig()
+	if len(cacheConfig.Drives) > 0 {
+		var err error
+		// initialize the new disk cache objects.
+		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
+		logger.FatalIf(err, "Unable to initialize disk caching")
+	}
+
+	// Load logger subsystem
+	loadLoggers()
+
+	// Re-enable logging
+	logger.Disable = false
+
+	// Create new IAM system.
+	globalIAMSys = NewIAMSys()
+	if globalEtcdClient != nil {
+		// Initialize IAM sys.
+		go globalIAMSys.Init(newObject)
+	}
+
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
+
+	// Initialize policy system.
 	go globalPolicySys.Init(newObject)
 
+	// Create new notification system.
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+
 	// Once endpoints are finalized, initialize the new object api.
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject
@@ -256,8 +287,5 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
 	}
 
-	// Reenable logging
-	logger.Disable = false
-
 	handleSignals()
 }

cmd/gateway-startup-msg.go

@@ -57,8 +57,10 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 	// Colorize the message and print.
 	logger.StartupMessage(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
-	logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
-	logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+	if isTerminal() {
+		logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+	}
 
 	if globalIsBrowserEnabled {
 		logger.StartupMessage(colorBlue("\nBrowser Access:"))

cmd/gateway/s3/gateway-s3.go

@@ -71,6 +71,9 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
      MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
+  LOGGER:
+     MINIO_LOGGER_HTTP_ENDPOINT: HTTP endpoint URL to log all incoming requests.
+
 EXAMPLES:
   1. Start minio gateway server for AWS S3 backend.
      $ export MINIO_ACCESS_KEY=accesskey
@@ -82,7 +85,13 @@ EXAMPLES:
      $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
      $ {{.HelpName}} https://play.minio.io:9000
 
-  3. Start minio gateway server for AWS S3 backend with edge caching enabled.
+  3. Start minio gateway server for AWS S3 backend logging all requests to http endpoint.
+     $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
+     $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+     $ export MINIO_LOGGER_HTTP_ENDPOINT="http://localhost:8000/"
+     $ {{.HelpName}} https://play.minio.io:9000
+
+  4. Start minio gateway server for AWS S3 backend with edge caching enabled.
      $ export MINIO_ACCESS_KEY=accesskey
      $ export MINIO_SECRET_KEY=secretkey
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
@@ -401,6 +410,9 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string,
 	if err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
 	}
+	// On success, populate the key & metadata so they are present in the notification
+	oi.Key = object
+	oi.Metadata = minio.ToMinioClientObjectInfoMetadata(metadata)
 
 	return minio.FromMinioClientObjectInfo(bucket, oi), nil
 }

cmd/gateway/s3/gateway-s3_test.go

@@ -50,6 +50,10 @@ func TestS3ToObjectError(t *testing.T) {
 			inputErr:    errResponse("InvalidBucketName"),
 			expectedErr: minio.BucketNameInvalid{},
 		},
+		{
+			inputErr:    errResponse("InvalidPart"),
+			expectedErr: minio.InvalidPart{},
+		},
 		{
 			inputErr:    errResponse("NoSuchBucketPolicy"),
 			expectedErr: minio.BucketPolicyNotFound{},

cmd/generic-handlers.go

@@ -28,6 +28,7 @@ import (
 	"github.com/minio/minio-go/pkg/set"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/handlers"
@@ -773,3 +774,17 @@ func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	}()
 	h.handler.ServeHTTP(w, r)
 }
+
+func setSSETLSHandler(h http.Handler) http.Handler { return sseTLSHandler{h} }
+
+// sseTLSHandler enforces certain rules for SSE requests which are made / must be made over TLS.
+type sseTLSHandler struct{ handler http.Handler }
+
+func (h sseTLSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Deny SSE-C requests if not made over TLS
+	if !globalIsSSL && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
+		writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
+		return
+	}
+	h.handler.ServeHTTP(w, r)
+}

cmd/generic-handlers_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"net/http"
+	"net/http/httptest"
 	"strconv"
 	"testing"
 
@@ -181,3 +182,39 @@ func TestContainsReservedMetadata(t *testing.T) {
 		}
 	}
 }
+
+var sseTLSHandlerTests = []struct {
+	Header            http.Header
+	IsTLS, ShouldFail bool
+}{
+	{Header: http.Header{}, IsTLS: false, ShouldFail: false},                                        // 0
+	{Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: false, ShouldFail: true}, // 1
+	{Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: true, ShouldFail: false}, // 2
+	{Header: http.Header{crypto.SSECKey: []string{""}}, IsTLS: true, ShouldFail: false},             // 3
+	{Header: http.Header{crypto.SSECopyAlgorithm: []string{""}}, IsTLS: false, ShouldFail: true},    // 4
+}
+
+func TestSSETLSHandler(t *testing.T) {
+	defer func(isSSL bool) { globalIsSSL = isSSL }(globalIsSSL) // reset globalIsSSL after test
+
+	var okHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}
+	for i, test := range sseTLSHandlerTests {
+		globalIsSSL = test.IsTLS
+
+		w := httptest.NewRecorder()
+		r := new(http.Request)
+		r.Header = test.Header
+
+		h := setSSETLSHandler(okHandler)
+		h.ServeHTTP(w, r)
+
+		switch {
+		case test.ShouldFail && w.Code == http.StatusOK:
+			t.Errorf("Test %d: should fail but status code is HTTP %d", i, w.Code)
+		case !test.ShouldFail && w.Code != http.StatusOK:
+			t.Errorf("Test %d: should not fail but status code is HTTP %d and not 200 OK", i, w.Code)
+		}
+	}
+}

cmd/globals.go

@@ -34,6 +34,8 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/dns"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
 )
 
 // minio configuration related constants.
@@ -81,6 +83,8 @@ const (
 	globalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
 	// Refresh interval to update in-memory bucket policy cache.
 	globalRefreshBucketPolicyInterval = 5 * time.Minute
+	// Refresh interval to update in-memory iam config cache.
+	globalRefreshIAMInterval = 5 * time.Minute
 
 	// Limit of location constraint XML for unauthenticted PUT bucket operations.
 	maxLocationConstraintSize = 3 * humanize.MiByte
@@ -132,6 +136,7 @@ var (
 
 	globalNotificationSys *NotificationSys
 	globalPolicySys       *PolicySys
+	globalIAMSys          *IAMSys
 
 	// CA root certificates, a nil value means system certs pool will be used
 	globalRootCAs *x509.CertPool
@@ -244,6 +249,12 @@ var (
 	// Some standard content-types which we strictly dis-allow for compression.
 	standardExcludeCompressContentTypes = []string{"video/*", "audio/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", " application/x-compress", "application/x-spoon"}
 
+	// Authorization validators list.
+	globalIAMValidators *validator.Validators
+
+	// OPA policy system.
+	globalPolicyOPA *iampolicy.Opa
+
 	// Add new variable global values here.
 )
 

cmd/http/bufconn_test.go

@@ -46,7 +46,8 @@ func TestBuffConnReadTimeout(t *testing.T) {
 
 		tcpConn, terr := tcpListener.AcceptTCP()
 		if terr != nil {
-			t.Fatalf("failed to accept new connection. %v", terr)
+			t.Errorf("failed to accept new connection. %v", terr)
+			return
 		}
 		bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second)
 		defer bufconn.Close()
@@ -55,11 +56,13 @@ func TestBuffConnReadTimeout(t *testing.T) {
 		var b = make([]byte, 12)
 		_, terr = bufconn.Read(b)
 		if terr != nil {
-			t.Fatalf("failed to read from client. %v", terr)
+			t.Errorf("failed to read from client. %v", terr)
+			return
 		}
 		received := string(b)
 		if received != "message one\n" {
-			t.Fatalf(`server: expected: "message one\n", got: %v`, received)
+			t.Errorf(`server: expected: "message one\n", got: %v`, received)
+			return
 		}
 
 		// Wait for more than read timeout to simulate processing.
@@ -67,17 +70,20 @@ func TestBuffConnReadTimeout(t *testing.T) {
 
 		_, terr = bufconn.Read(b)
 		if terr != nil {
-			t.Fatalf("failed to read from client. %v", terr)
+			t.Errorf("failed to read from client. %v", terr)
+			return
 		}
 		received = string(b)
 		if received != "message two\n" {
-			t.Fatalf(`server: expected: "message two\n", got: %v`, received)
+			t.Errorf(`server: expected: "message two\n", got: %v`, received)
+			return
 		}
 
 		// Send a response.
 		_, terr = io.WriteString(bufconn, "messages received\n")
 		if terr != nil {
-			t.Fatalf("failed to write to client. %v", terr)
+			t.Errorf("failed to write to client. %v", terr)
+			return
 		}
 
 		// Removes all deadlines if any.

cmd/iam.go

@@ -0,0 +1,699 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+const (
+	// IAM configuration directory.
+	iamConfigPrefix = minioConfigPrefix + "/iam"
+
+	// IAM users directory.
+	iamConfigUsersPrefix = iamConfigPrefix + "/users/"
+
+	// IAM policies directory.
+	iamConfigPoliciesPrefix = iamConfigPrefix + "/policies/"
+
+	// IAM sts directory.
+	iamConfigSTSPrefix = iamConfigPrefix + "/sts/"
+
+	// IAM identity file which captures identity credentials.
+	iamIdentityFile = "identity.json"
+
+	// IAM policy file which provides policies for each users.
+	iamPolicyFile = "policy.json"
+)
+
+// IAMSys - config system.
+type IAMSys struct {
+	sync.RWMutex
+	iamUsersMap        map[string]auth.Credentials
+	iamPolicyMap       map[string]string
+	iamCannedPolicyMap map[string]iampolicy.Policy
+}
+
+// Load - load iam.json
+func (sys *IAMSys) Load(objAPI ObjectLayer) error {
+	return sys.Init(objAPI)
+}
+
+// Init - initializes config system from iam.json
+func (sys *IAMSys) Init(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+
+	defer func() {
+		// Refresh IAMSys in background.
+		go func() {
+			ticker := time.NewTicker(globalRefreshIAMInterval)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-globalServiceDoneCh:
+					return
+				case <-ticker.C:
+					sys.refresh(objAPI)
+				}
+			}
+		}()
+	}()
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing IAM needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			// Load IAMSys once during boot.
+			if err := sys.refresh(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for IAM subsystem to be initialized..")
+					continue
+				}
+				return err
+			}
+			return nil
+		}
+	}
+}
+
+// DeleteCannedPolicy - deletes a canned policy.
+func (sys *IAMSys) DeleteCannedPolicy(policyName string) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	if policyName == "" {
+		return errInvalidArgument
+	}
+
+	var err error
+	configFile := pathJoin(iamConfigPoliciesPrefix, policyName, iamPolicyFile)
+	if globalEtcdClient != nil {
+		err = deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
+	} else {
+		err = deleteConfig(context.Background(), objectAPI, configFile)
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	delete(sys.iamCannedPolicyMap, policyName)
+	return err
+}
+
+// ListCannedPolicies - lists all canned policies.
+func (sys *IAMSys) ListCannedPolicies() (map[string][]byte, error) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+
+	var cannedPolicyMap = make(map[string][]byte)
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	for k, v := range sys.iamCannedPolicyMap {
+		data, err := json.Marshal(v)
+		if err != nil {
+			return nil, err
+		}
+		cannedPolicyMap[k] = data
+	}
+
+	return cannedPolicyMap, nil
+}
+
+// SetCannedPolicy - sets a new canned policy.
+func (sys *IAMSys) SetCannedPolicy(policyName string, p iampolicy.Policy) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	if p.IsEmpty() || policyName == "" {
+		return errInvalidArgument
+	}
+
+	configFile := pathJoin(iamConfigPoliciesPrefix, policyName, iamPolicyFile)
+	data, err := json.Marshal(p)
+	if err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+	if err != nil {
+		return err
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.iamCannedPolicyMap[policyName] = p
+
+	return nil
+}
+
+// SetUserPolicy - sets policy to given user name.
+func (sys *IAMSys) SetUserPolicy(accessKey, policyName string) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	if _, ok := sys.iamUsersMap[accessKey]; !ok {
+		return errNoSuchUser
+	}
+
+	if _, ok := sys.iamCannedPolicyMap[policyName]; !ok {
+		return errNoSuchPolicy
+	}
+
+	data, err := json.Marshal(policyName)
+	if err != nil {
+		return err
+	}
+
+	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile)
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+	if err != nil {
+		return err
+	}
+
+	sys.iamPolicyMap[accessKey] = policyName
+	return nil
+}
+
+// DeleteUser - set user credentials.
+func (sys *IAMSys) DeleteUser(accessKey string) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	var err error
+	pFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile)
+	iFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
+	if globalEtcdClient != nil {
+		// It is okay to ingnore errors when deleting policy.json for the user.
+		_ = deleteConfigEtcd(context.Background(), globalEtcdClient, pFile)
+		err = deleteConfigEtcd(context.Background(), globalEtcdClient, iFile)
+	} else {
+		// It is okay to ingnore errors when deleting policy.json for the user.
+		_ = deleteConfig(context.Background(), objectAPI, pFile)
+		err = deleteConfig(context.Background(), objectAPI, iFile)
+	}
+
+	//
+	switch err.(type) {
+	case ObjectNotFound:
+		err = errNoSuchUser
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	delete(sys.iamUsersMap, accessKey)
+	delete(sys.iamPolicyMap, accessKey)
+
+	return err
+}
+
+// SetTempUser - set temporary user credentials, these credentials have an expiry.
+func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	configFile := pathJoin(iamConfigSTSPrefix, accessKey, iamIdentityFile)
+	data, err := json.Marshal(cred)
+	if err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.iamUsersMap[accessKey] = cred
+	return nil
+}
+
+// ListUsers - list all users.
+func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+
+	var users = make(map[string]madmin.UserInfo)
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	for k, v := range sys.iamUsersMap {
+		users[k] = madmin.UserInfo{
+			PolicyName: sys.iamPolicyMap[k],
+			Status:     madmin.AccountStatus(v.Status),
+		}
+	}
+
+	return users, nil
+}
+
+// SetUserStatus - sets current user status, supports disabled or enabled.
+func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	if status != madmin.AccountEnabled && status != madmin.AccountDisabled {
+		return errInvalidArgument
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	cred, ok := sys.iamUsersMap[accessKey]
+	if !ok {
+		return errNoSuchUser
+	}
+
+	uinfo := madmin.UserInfo{
+		SecretKey: cred.SecretKey,
+		Status:    status,
+	}
+
+	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
+	data, err := json.Marshal(uinfo)
+	if err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	sys.iamUsersMap[accessKey] = auth.Credentials{
+		AccessKey: accessKey,
+		SecretKey: uinfo.SecretKey,
+		Status:    string(uinfo.Status),
+	}
+
+	return nil
+}
+
+// SetUser - set user credentials.
+func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
+	data, err := json.Marshal(uinfo)
+	if err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.iamUsersMap[accessKey] = auth.Credentials{
+		AccessKey: accessKey,
+		SecretKey: uinfo.SecretKey,
+		Status:    string(uinfo.Status),
+	}
+
+	return nil
+}
+
+// GetUser - get user credentials
+func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
+	sys.RLock()
+	defer sys.RUnlock()
+
+	cred, ok = sys.iamUsersMap[accessKey]
+	return cred, ok && cred.IsValid()
+}
+
+// IsAllowed - checks given policy args is allowed to continue the Rest API.
+func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
+	sys.RLock()
+	defer sys.RUnlock()
+
+	// If opa is configured, use OPA always.
+	if globalPolicyOPA != nil {
+		return globalPolicyOPA.IsAllowed(args)
+	}
+
+	// If policy is available for given user, check the policy.
+	if name, found := sys.iamPolicyMap[args.AccountName]; found {
+		p, ok := sys.iamCannedPolicyMap[name]
+		return ok && p.IsAllowed(args)
+	}
+
+	// As policy is not available and OPA is not configured, return the owner value.
+	return args.IsOwner
+}
+
+var defaultContextTimeout = 5 * time.Minute
+
+// Similar to reloadUsers but updates users, policies maps from etcd server,
+func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policyMap map[string]string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+	defer cancel()
+	if err != nil {
+		return err
+	}
+	// No users are created yet.
+	if r.Count == 0 {
+		return nil
+	}
+
+	users := set.NewStringSet()
+	for _, kv := range r.Kvs {
+		// Extract user by stripping off the `prefix` value as suffix,
+		// then strip off the remaining basename to obtain the prefix
+		// value, usually in the following form.
+		//
+		//  key := "config/iam/users/newuser/identity.json"
+		//  prefix := "config/iam/users/"
+		//  v := trim(trim(key, prefix), base(key)) == "newuser"
+		//
+		user := path.Clean(strings.TrimSuffix(strings.TrimPrefix(string(kv.Key), prefix), path.Base(string(kv.Key))))
+		if !users.Contains(user) {
+			users.Add(user)
+		}
+	}
+
+	// Reload config and policies for all users.
+	for _, user := range users.ToSlice() {
+		idFile := pathJoin(prefix, user, iamIdentityFile)
+		pFile := pathJoin(prefix, user, iamPolicyFile)
+		cdata, cerr := readConfigEtcd(ctx, globalEtcdClient, idFile)
+		pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile)
+		if cerr != nil && cerr != errConfigNotFound {
+			return cerr
+		}
+		if perr != nil && perr != errConfigNotFound {
+			return perr
+		}
+		if cerr == errConfigNotFound && perr == errConfigNotFound {
+			continue
+		}
+		if cerr == nil {
+			var cred auth.Credentials
+			if err = json.Unmarshal(cdata, &cred); err != nil {
+				return err
+			}
+			cred.AccessKey = user
+			if cred.IsExpired() {
+				deleteConfigEtcd(ctx, globalEtcdClient, idFile)
+				deleteConfigEtcd(ctx, globalEtcdClient, pFile)
+				continue
+			}
+			usersMap[cred.AccessKey] = cred
+		}
+		if perr == nil {
+			var policyName string
+			if err = json.Unmarshal(pdata, &policyName); err != nil {
+				return err
+			}
+			policyMap[user] = policyName
+		}
+	}
+	return nil
+}
+
+func reloadEtcdPolicies(prefix string, cannedPolicyMap map[string]iampolicy.Policy) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+	defer cancel()
+	if err != nil {
+		return err
+	}
+	// No users are created yet.
+	if r.Count == 0 {
+		return nil
+	}
+
+	policies := set.NewStringSet()
+	for _, kv := range r.Kvs {
+		// Extract policy by stripping off the `prefix` value as suffix,
+		// then strip off the remaining basename to obtain the prefix
+		// value, usually in the following form.
+		//
+		//  key := "config/iam/policies/newpolicy/identity.json"
+		//  prefix := "config/iam/policies/"
+		//  v := trim(trim(key, prefix), base(key)) == "newpolicy"
+		//
+		policyName := path.Clean(strings.TrimSuffix(strings.TrimPrefix(string(kv.Key), prefix), path.Base(string(kv.Key))))
+		if !policies.Contains(policyName) {
+			policies.Add(policyName)
+		}
+	}
+
+	// Reload config and policies for all policys.
+	for _, policyName := range policies.ToSlice() {
+		pFile := pathJoin(prefix, policyName, iamPolicyFile)
+		pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile)
+		if perr != nil {
+			return perr
+		}
+		var p iampolicy.Policy
+		if err = json.Unmarshal(pdata, &p); err != nil {
+			return err
+		}
+		cannedPolicyMap[policyName] = p
+	}
+	return nil
+}
+
+func reloadPolicies(objectAPI ObjectLayer, prefix string, cannedPolicyMap map[string]iampolicy.Policy) error {
+	marker := ""
+	for {
+		var lo ListObjectsInfo
+		var err error
+		lo, err = objectAPI.ListObjects(context.Background(), minioMetaBucket, prefix, marker, "/", 1000)
+		if err != nil {
+			return err
+		}
+		marker = lo.NextMarker
+		for _, prefix := range lo.Prefixes {
+			pFile := pathJoin(prefix, iamPolicyFile)
+			pdata, perr := readConfig(context.Background(), objectAPI, pFile)
+			if perr != nil {
+				return perr
+			}
+			var p iampolicy.Policy
+			if err = json.Unmarshal(pdata, &p); err != nil {
+				return err
+			}
+			cannedPolicyMap[path.Base(prefix)] = p
+		}
+		if !lo.IsTruncated {
+			break
+		}
+	}
+	return nil
+
+}
+
+// reloadUsers reads an updates users, policies from object layer into user and policy maps.
+func reloadUsers(objectAPI ObjectLayer, prefix string, usersMap map[string]auth.Credentials, policyMap map[string]string) error {
+	marker := ""
+	for {
+		var lo ListObjectsInfo
+		var err error
+		lo, err = objectAPI.ListObjects(context.Background(), minioMetaBucket, prefix, marker, "/", 1000)
+		if err != nil {
+			return err
+		}
+		marker = lo.NextMarker
+		for _, prefix := range lo.Prefixes {
+			idFile := pathJoin(prefix, iamIdentityFile)
+			pFile := pathJoin(prefix, iamPolicyFile)
+			cdata, cerr := readConfig(context.Background(), objectAPI, idFile)
+			pdata, perr := readConfig(context.Background(), objectAPI, pFile)
+			if cerr != nil && cerr != errConfigNotFound {
+				return cerr
+			}
+			if perr != nil && perr != errConfigNotFound {
+				return perr
+			}
+			if cerr == errConfigNotFound && perr == errConfigNotFound {
+				continue
+			}
+			if cerr == nil {
+				var cred auth.Credentials
+				if err = json.Unmarshal(cdata, &cred); err != nil {
+					return err
+				}
+				cred.AccessKey = path.Base(prefix)
+				if cred.IsExpired() {
+					// Delete expired identity.
+					objectAPI.DeleteObject(context.Background(), minioMetaBucket, idFile)
+					// Delete expired identity policy.
+					objectAPI.DeleteObject(context.Background(), minioMetaBucket, pFile)
+					continue
+				}
+				usersMap[cred.AccessKey] = cred
+			}
+			if perr == nil {
+				var policyName string
+				if err = json.Unmarshal(pdata, &policyName); err != nil {
+					return err
+				}
+				policyMap[path.Base(prefix)] = policyName
+			}
+		}
+		if !lo.IsTruncated {
+			break
+		}
+	}
+	return nil
+}
+
+// Set default canned policies only if not already overridden by users.
+func setDefaultCannedPolicies(policies map[string]iampolicy.Policy) {
+	_, ok := policies["writeonly"]
+	if !ok {
+		policies["writeonly"] = iampolicy.WriteOnly
+	}
+	_, ok = policies["readonly"]
+	if !ok {
+		policies["readonly"] = iampolicy.ReadOnly
+	}
+	_, ok = policies["readwrite"]
+	if !ok {
+		policies["readwrite"] = iampolicy.ReadWrite
+	}
+}
+
+// Refresh IAMSys.
+func (sys *IAMSys) refresh(objAPI ObjectLayer) error {
+	iamUsersMap := make(map[string]auth.Credentials)
+	iamPolicyMap := make(map[string]string)
+	iamCannedPolicyMap := make(map[string]iampolicy.Policy)
+
+	if globalEtcdClient != nil {
+		if err := reloadEtcdPolicies(iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadEtcdUsers(iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadEtcdUsers(iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+	} else {
+		if err := reloadPolicies(objAPI, iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+		if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
+			return err
+		}
+	}
+
+	// Sets default canned policies, if none set.
+	setDefaultCannedPolicies(iamCannedPolicyMap)
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.iamUsersMap = iamUsersMap
+	sys.iamPolicyMap = iamPolicyMap
+	sys.iamCannedPolicyMap = iamCannedPolicyMap
+
+	return nil
+}
+
+// NewIAMSys - creates new config system object.
+func NewIAMSys() *IAMSys {
+	return &IAMSys{
+		iamUsersMap:        make(map[string]auth.Credentials),
+		iamPolicyMap:       make(map[string]string),
+		iamCannedPolicyMap: make(map[string]iampolicy.Policy),
+	}
+}

cmd/jwt.go

@@ -49,7 +49,33 @@ var (
 	errNoAuthToken          = errors.New("JWT token missing")
 )
 
-func authenticateJWT(accessKey, secretKey string, expiry time.Duration) (string, error) {
+func authenticateJWTUsers(accessKey, secretKey string, expiry time.Duration) (string, error) {
+	passedCredential, err := auth.CreateCredentials(accessKey, secretKey)
+	if err != nil {
+		return "", err
+	}
+
+	serverCred := globalServerConfig.GetCredential()
+	if serverCred.AccessKey != passedCredential.AccessKey {
+		var ok bool
+		serverCred, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return "", errInvalidAccessKeyID
+		}
+	}
+
+	if !serverCred.Equal(passedCredential) {
+		return "", errAuthentication
+	}
+
+	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{
+		ExpiresAt: UTCNow().Add(expiry).Unix(),
+		Subject:   accessKey,
+	})
+	return jwt.SignedString([]byte(serverCred.SecretKey))
+}
+
+func authenticateJWTAdmin(accessKey, secretKey string, expiry time.Duration) (string, error) {
 	passedCredential, err := auth.CreateCredentials(accessKey, secretKey)
 	if err != nil {
 		return "", err
@@ -73,68 +99,133 @@ func authenticateJWT(accessKey, secretKey string, expiry time.Duration) (string,
 }
 
 func authenticateNode(accessKey, secretKey string) (string, error) {
-	return authenticateJWT(accessKey, secretKey, defaultInterNodeJWTExpiry)
+	return authenticateJWTAdmin(accessKey, secretKey, defaultInterNodeJWTExpiry)
 }
 
 func authenticateWeb(accessKey, secretKey string) (string, error) {
-	return authenticateJWT(accessKey, secretKey, defaultJWTExpiry)
+	return authenticateJWTUsers(accessKey, secretKey, defaultJWTExpiry)
 }
 
 func authenticateURL(accessKey, secretKey string) (string, error) {
-	return authenticateJWT(accessKey, secretKey, defaultURLJWTExpiry)
+	return authenticateJWTUsers(accessKey, secretKey, defaultURLJWTExpiry)
 }
 
-func keyFuncCallback(jwtToken *jwtgo.Token) (interface{}, error) {
+func stsTokenCallback(jwtToken *jwtgo.Token) (interface{}, error) {
 	if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
 		return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
 	}
 
-	return []byte(globalServerConfig.GetCredential().SecretKey), nil
+	if err := jwtToken.Claims.Valid(); err != nil {
+		return nil, errAuthentication
+	}
+	if claims, ok := jwtToken.Claims.(jwtgo.MapClaims); ok {
+		accessKey, ok := claims["accessKey"].(string)
+		if !ok {
+			return nil, errInvalidAccessKeyID
+		}
+		if accessKey == globalServerConfig.GetCredential().AccessKey {
+			return []byte(globalServerConfig.GetCredential().SecretKey), nil
+		}
+		if globalIAMSys == nil {
+			return nil, errInvalidAccessKeyID
+		}
+		_, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return nil, errInvalidAccessKeyID
+		}
+		return []byte(globalServerConfig.GetCredential().SecretKey), nil
+	}
+	return nil, errAuthentication
 }
 
-func isAuthTokenValid(tokenString string) bool {
-	if tokenString == "" {
-		return false
+// Callback function used for parsing
+func webTokenCallback(jwtToken *jwtgo.Token) (interface{}, error) {
+	if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
+		return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
 	}
-	var claims jwtgo.StandardClaims
-	jwtToken, err := jwtgo.ParseWithClaims(tokenString, &claims, keyFuncCallback)
+
+	if err := jwtToken.Claims.Valid(); err != nil {
+		return nil, errAuthentication
+	}
+
+	if claims, ok := jwtToken.Claims.(*jwtgo.StandardClaims); ok {
+		if claims.Subject == globalServerConfig.GetCredential().AccessKey {
+			return []byte(globalServerConfig.GetCredential().SecretKey), nil
+		}
+		if globalIAMSys == nil {
+			return nil, errInvalidAccessKeyID
+		}
+		cred, ok := globalIAMSys.GetUser(claims.Subject)
+		if !ok {
+			return nil, errInvalidAccessKeyID
+		}
+		return []byte(cred.SecretKey), nil
+	}
+
+	return nil, errAuthentication
+}
+
+func parseJWTWithClaims(tokenString string, claims jwtgo.Claims) (*jwtgo.Token, error) {
+	p := &jwtgo.Parser{
+		SkipClaimsValidation: true,
+	}
+	jwtToken, err := p.ParseWithClaims(tokenString, claims, webTokenCallback)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return false
+		switch e := err.(type) {
+		case *jwtgo.ValidationError:
+			if e.Inner == nil {
+				return nil, errAuthentication
+			}
+			return nil, e.Inner
+		}
+		return nil, errAuthentication
 	}
-	if err = claims.Valid(); err != nil {
-		logger.LogIf(context.Background(), err)
-		return false
-	}
-	return jwtToken.Valid && claims.Subject == globalServerConfig.GetCredential().AccessKey
+	return jwtToken, nil
 }
 
-func isHTTPRequestValid(req *http.Request) bool {
-	return webRequestAuthenticate(req) == nil
+func isAuthTokenValid(token string) bool {
+	_, _, err := webTokenAuthenticate(token)
+	return err == nil
+}
+
+func webTokenAuthenticate(token string) (jwtgo.StandardClaims, bool, error) {
+	var claims = jwtgo.StandardClaims{}
+	if token == "" {
+		return claims, false, errNoAuthToken
+	}
+
+	jwtToken, err := parseJWTWithClaims(token, &claims)
+	if err != nil {
+		return claims, false, err
+	}
+	if !jwtToken.Valid {
+		return claims, false, errAuthentication
+	}
+	owner := claims.Subject == globalServerConfig.GetCredential().AccessKey
+	return claims, owner, nil
 }
 
 // Check if the request is authenticated.
 // Returns nil if the request is authenticated. errNoAuthToken if token missing.
 // Returns errAuthentication for all other errors.
-func webRequestAuthenticate(req *http.Request) error {
-	var claims jwtgo.StandardClaims
-	jwtToken, err := jwtreq.ParseFromRequestWithClaims(req, jwtreq.AuthorizationHeaderExtractor, &claims, keyFuncCallback)
+func webRequestAuthenticate(req *http.Request) (jwtgo.StandardClaims, bool, error) {
+	var claims = jwtgo.StandardClaims{}
+	tokStr, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(req)
 	if err != nil {
 		if err == jwtreq.ErrNoTokenInRequest {
-			return errNoAuthToken
+			return claims, false, errNoAuthToken
 		}
-		return errAuthentication
+		return claims, false, err
 	}
-	if err = claims.Valid(); err != nil {
-		return errAuthentication
-	}
-	if claims.Subject != globalServerConfig.GetCredential().AccessKey {
-		return errInvalidAccessKeyID
+	jwtToken, err := parseJWTWithClaims(tokStr, &claims)
+	if err != nil {
+		return claims, false, err
 	}
 	if !jwtToken.Valid {
-		return errAuthentication
+		return claims, false, errAuthentication
 	}
-	return nil
+	owner := claims.Subject == globalServerConfig.GetCredential().AccessKey
+	return claims, owner, nil
 }
 
 func newAuthToken() string {

cmd/jwt_test.go

@@ -142,7 +142,7 @@ func TestWebRequestAuthenticate(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		gotErr := webRequestAuthenticate(testCase.req)
+		_, _, gotErr := webRequestAuthenticate(testCase.req)
 		if testCase.expectedErr != gotErr {
 			t.Errorf("Test %d, expected err %s, got %s", i+1, testCase.expectedErr, gotErr)
 		}

cmd/logger/console.go

@@ -27,7 +27,11 @@ import (
 // in plain or json format to the standard output.
 type ConsoleTarget struct{}
 
-func (c *ConsoleTarget) send(entry logEntry) error {
+func (c *ConsoleTarget) send(e interface{}) error {
+	entry, ok := e.(logEntry)
+	if !ok {
+		return fmt.Errorf("Uexpected log entry structure %#v", e)
+	}
 	if jsonFlag {
 		logJSON, err := json.Marshal(&entry)
 		if err != nil {

cmd/logger/http.go

@@ -32,7 +32,8 @@ import (
 // is returned to the caller.
 type HTTPTarget struct {
 	// Channel of log entries
-	logCh chan logEntry
+	logCh chan interface{}
+
 	// HTTP(s) endpoint
 	endpoint string
 	client   http.Client
@@ -72,14 +73,14 @@ func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
 		client: http.Client{
 			Transport: transport,
 		},
-		logCh: make(chan logEntry, 10000),
+		logCh: make(chan interface{}, 10000),
 	}
 
 	h.startHTTPLogger()
 	return &h
 }
 
-func (h *HTTPTarget) send(entry logEntry) error {
+func (h *HTTPTarget) send(entry interface{}) error {
 	select {
 	case h.logCh <- entry:
 	default:

cmd/logger/logger.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"go/build"
+	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -341,6 +342,55 @@ func logIf(ctx context.Context, err error) {
 	}
 }
 
+type auditEntry struct {
+	DeploymentID string            `json:"deploymentid,omitempty"`
+	Time         string            `json:"time"`
+	API          *api              `json:"api,omitempty"`
+	RemoteHost   string            `json:"remotehost,omitempty"`
+	RequestID    string            `json:"requestID,omitempty"`
+	UserAgent    string            `json:"userAgent,omitempty"`
+	Metadata     map[string]string `json:"metadata,omitempty"`
+}
+
+// AuditLog - logs audit logs to all targets.
+func AuditLog(ctx context.Context, r *http.Request) {
+	if Disable {
+		return
+	}
+
+	req := GetReqInfo(ctx)
+	if req == nil {
+		req = &ReqInfo{API: "SYSTEM"}
+	}
+
+	API := "SYSTEM"
+	if req.API != "" {
+		API = req.API
+	}
+
+	tags := make(map[string]string)
+	for _, entry := range req.GetTags() {
+		tags[entry.Key] = entry.Val
+	}
+
+	entry := auditEntry{
+		DeploymentID: deploymentID,
+		RemoteHost:   req.RemoteHost,
+		RequestID:    req.RequestID,
+		UserAgent:    req.UserAgent,
+		Time:         time.Now().UTC().Format(time.RFC3339Nano),
+		API:          &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
+		Metadata:     tags,
+	}
+
+	// Send audit logs only to http targets.
+	for _, t := range Targets {
+		if _, ok := t.(*HTTPTarget); ok {
+			t.send(entry)
+		}
+	}
+}
+
 // ErrCritical is the value panic'd whenever CriticalIf is called.
 var ErrCritical struct{}
 
@@ -450,10 +500,9 @@ func (f fatalMsg) pretty(msg string, args ...interface{}) {
 	os.Exit(1)
 }
 
-var info infoMsg
+type infoMsg struct{}
 
-type infoMsg struct {
-}
+var info infoMsg
 
 func (i infoMsg) json(msg string, args ...interface{}) {
 	logJSON, err := json.Marshal(&logEntry{

cmd/logger/targets.go

@@ -20,7 +20,7 @@ package logger
 // a single log entry and send it to the log target
 //   e.g. send the log to a http server
 type LoggingTarget interface {
-	send(entry logEntry) error
+	send(entry interface{}) error
 }
 
 // Targets is the set of enabled loggers

cmd/namespace-lock_test.go

@@ -218,7 +218,8 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
 		// Try to claim lock again.
 		anotherLock := globalNSMutex.NewNSLock("bucket", "object")
 		if anotherLock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
-			t.Fatalf("Failed to get lock")
+			t.Errorf("Failed to get lock")
+			return
 		}
 		// And signal success.
 		ch <- struct{}{}

cmd/notification.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"encoding/xml"
@@ -241,20 +242,19 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
+	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
 		return err
 	}
-	defer objLock.Unlock()
+	defer objLock.RUnlock()
 
-	reader, e := readConfig(ctx, objAPI, configFile)
+	configData, e := readConfig(ctx, objAPI, configFile)
 	if e != nil && !IsErrIgnored(e, errDiskNotFound, errConfigNotFound) {
 		return e
 	}
 
 	listenerList := []ListenBucketNotificationArgs{}
-	if reader != nil {
-		err := json.NewDecoder(reader).Decode(&listenerList)
-		if err != nil {
+	if configData != nil {
+		if err := json.Unmarshal(configData, &listenerList); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
@@ -265,7 +265,6 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 		return nil
 	}
 
-	activeListenerList := []ListenBucketNotificationArgs{}
 	for _, args := range listenerList {
 		found, err := isLocalHost(args.Addr.Name)
 		if err != nil {
@@ -301,16 +300,31 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 			logger.LogIf(ctx, err)
 			return err
 		}
-		activeListenerList = append(activeListenerList, args)
 	}
 
-	data, err := json.Marshal(activeListenerList)
+	return nil
+}
+
+func (sys *NotificationSys) refresh(objAPI ObjectLayer) error {
+	buckets, err := objAPI.ListBuckets(context.Background())
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return err
 	}
-
-	return saveConfig(objAPI, configFile, data)
+	for _, bucket := range buckets {
+		ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name})
+		config, err := readNotificationConfig(ctx, objAPI, bucket.Name)
+		if err != nil && err != errNoSuchNotifications {
+			return err
+		}
+		if err == errNoSuchNotifications {
+			continue
+		}
+		sys.AddRulesMap(bucket.Name, config.ToRulesMap())
+		if err = sys.initListeners(ctx, objAPI, bucket.Name); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // Init - initializes notification system from notification.xml and listener.json of all buckets.
@@ -319,28 +333,29 @@ func (sys *NotificationSys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	buckets, err := objAPI.ListBuckets(context.Background())
-	if err != nil {
-		return err
-	}
+	doneCh := make(chan struct{})
+	defer close(doneCh)
 
-	for _, bucket := range buckets {
-		ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name})
-		config, err := readNotificationConfig(ctx, objAPI, bucket.Name)
-		if err != nil {
-			if !IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+	// Initializing notification needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			if err := sys.refresh(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for notification subsystem to be initialized..")
+					continue
+				}
 				return err
 			}
-		} else {
-			sys.AddRulesMap(bucket.Name, config.ToRulesMap())
-		}
-
-		if err = sys.initListeners(ctx, objAPI, bucket.Name); err != nil {
-			return err
+			return nil
 		}
 	}
-
-	return nil
 }
 
 // AddRulesMap - adds rules map for bucket name.
@@ -433,6 +448,7 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
 	sys.RLock()
 	targetIDSet := sys.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name)
 	sys.RUnlock()
+
 	if len(targetIDSet) == 0 {
 		return nil
 	}
@@ -550,7 +566,7 @@ func sendEvent(args eventArgs) {
 func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) (*event.Config, error) {
 	// Construct path to notification.xml for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
-	reader, err := readConfig(ctx, objAPI, configFile)
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
 		if err == errConfigNotFound {
 			err = errNoSuchNotifications
@@ -559,19 +575,19 @@ func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName
 		return nil, err
 	}
 
-	config, err := event.ParseConfig(reader, globalServerConfig.GetRegion(), globalNotificationSys.targetList)
+	config, err := event.ParseConfig(bytes.NewReader(configData), globalServerConfig.GetRegion(), globalNotificationSys.targetList)
 	logger.LogIf(ctx, err)
 	return config, err
 }
 
-func saveNotificationConfig(objAPI ObjectLayer, bucketName string, config *event.Config) error {
+func saveNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, config *event.Config) error {
 	data, err := xml.Marshal(config)
 	if err != nil {
 		return err
 	}
 
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
-	return saveConfig(objAPI, configFile, data)
+	return saveConfig(ctx, objAPI, configFile, data)
 }
 
 // SaveListener - saves HTTP client currently listening for events to listener.json.
@@ -596,14 +612,14 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
 	}
 	defer objLock.Unlock()
 
-	reader, err := readConfig(ctx, objAPI, configFile)
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil && !IsErrIgnored(err, errDiskNotFound, errConfigNotFound) {
 		return err
 	}
 
 	listenerList := []ListenBucketNotificationArgs{}
-	if reader != nil {
-		if err = json.NewDecoder(reader).Decode(&listenerList); err != nil {
+	if configData != nil {
+		if err = json.Unmarshal(configData, &listenerList); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
@@ -622,7 +638,7 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
 		return err
 	}
 
-	return saveConfig(objAPI, configFile, data)
+	return saveConfig(ctx, objAPI, configFile, data)
 }
 
 // RemoveListener - removes HTTP client currently listening for events from listener.json.
@@ -647,14 +663,14 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
 	}
 	defer objLock.Unlock()
 
-	reader, err := readConfig(ctx, objAPI, configFile)
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil && !IsErrIgnored(err, errDiskNotFound, errConfigNotFound) {
 		return err
 	}
 
 	listenerList := []ListenBucketNotificationArgs{}
-	if reader != nil {
-		if err = json.NewDecoder(reader).Decode(&listenerList); err != nil {
+	if configData != nil {
+		if err = json.Unmarshal(configData, &listenerList); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
@@ -681,5 +697,5 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
 		return err
 	}
 
-	return saveConfig(objAPI, configFile, data)
+	return saveConfig(ctx, objAPI, configFile, data)
 }

cmd/object-api-datatypes.go

@@ -109,8 +109,12 @@ type ObjectInfo struct {
 	Writer       io.WriteCloser `json:"-"`
 	Reader       *hash.Reader   `json:"-"`
 	metadataOnly bool
+
 	// Date and time when the object was last accessed.
 	AccTime time.Time
+
+	// backendType indicates which backend filled this structure
+	backendType BackendType
 }
 
 // ListPartsInfo - represents list of all parts.

cmd/object-api-errors.go

@@ -378,26 +378,8 @@ func (e BackendDown) Error() string {
 	return "Backend down"
 }
 
-// isErrIncompleteBody - Check if error type is IncompleteBody.
-func isErrIncompleteBody(err error) bool {
-	_, ok := err.(IncompleteBody)
-	return ok
-}
-
 // isErrObjectNotFound - Check if error type is ObjectNotFound.
 func isErrObjectNotFound(err error) bool {
 	_, ok := err.(ObjectNotFound)
 	return ok
 }
-
-// isInsufficientReadQuorum - Check if error type is InsufficientReadQuorum.
-func isInsufficientReadQuorum(err error) bool {
-	_, ok := err.(InsufficientReadQuorum)
-	return ok
-}
-
-// isInsufficientWriteQuorum - Check if error type is InsufficientWriteQuorum.
-func isInsufficientWriteQuorum(err error) bool {
-	_, ok := err.(InsufficientWriteQuorum)
-	return ok
-}

cmd/object-api-putobject_test.go

@@ -159,6 +159,16 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 		// Test case 30
 		// valid data with X-Amz-Meta- meta
 		{bucket, object, data, map[string]string{"X-Amz-Meta-AppID": "a42"}, "", int64(len(data)), getMD5Hash(data), nil},
+
+		// Test case 31
+		// Put an empty object with a trailing slash
+		{bucket, "emptydir/", []byte{}, nil, "", 0, getMD5Hash([]byte{}), nil},
+		// Test case 32
+		// Put an object inside the empty directory
+		{bucket, "emptydir/" + object, data, nil, "", int64(len(data)), getMD5Hash(data), nil},
+		// Test case 33
+		// Put the empty object with a trailing slash again (refer to Test case 31), this needs to succeed
+		{bucket, "emptydir/", []byte{}, nil, "", 0, getMD5Hash([]byte{}), nil},
 	}
 
 	for i, testCase := range testCases {
@@ -171,7 +181,7 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 		}
 		// Failed as expected, but does it fail for the expected reason.
 		if actualErr != nil && actualErr != testCase.expectedError {
-			t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error())
+			t.Errorf("Test %d: %s: Expected to fail with error \"%v\", but instead failed with error \"%v\" instead.", i+1, instanceType, testCase.expectedError, actualErr)
 		}
 		// Test passes as expected, but the output values are verified for correctness here.
 		if actualErr == nil {

cmd/object-handlers.go

@@ -78,13 +78,18 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
 func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SelectObject")
 
+	defer logger.AuditLog(ctx, r)
+
 	// Fetch object stat info.
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-
+	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
+		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		return
+	}
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -138,8 +143,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		writeErrorResponse(w, ErrEmptyRequestBody, r.URL)
 		return
 	}
-
-	var selectReq ObjectSelectRequest
+	var selectReq s3select.ObjectSelectRequest
 	if err := xmlDecoder(r.Body, &selectReq, r.ContentLength); err != nil {
 		writeErrorResponse(w, ErrMalformedXML, r.URL)
 		return
@@ -168,22 +172,21 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 
 	objInfo := gr.ObjInfo
 
-	if selectReq.InputSerialization.CompressionType == SelectCompressionGZIP {
+	if selectReq.InputSerialization.CompressionType == s3select.SelectCompressionGZIP {
 		if !strings.Contains(objInfo.ContentType, "gzip") {
 			writeErrorResponse(w, ErrInvalidDataSource, r.URL)
 			return
 		}
 	}
-	if selectReq.InputSerialization.CompressionType == SelectCompressionBZIP {
+	if selectReq.InputSerialization.CompressionType == s3select.SelectCompressionBZIP {
 		if !strings.Contains(objInfo.ContentType, "bzip") {
 			writeErrorResponse(w, ErrInvalidDataSource, r.URL)
 			return
 		}
 	}
-	if selectReq.InputSerialization.CompressionType == SelectCompressionNONE ||
-		selectReq.InputSerialization.CompressionType == "" {
-		selectReq.InputSerialization.CompressionType = SelectCompressionNONE
-		if !strings.Contains(objInfo.ContentType, "text/csv") {
+	if selectReq.InputSerialization.CompressionType == "" {
+		selectReq.InputSerialization.CompressionType = s3select.SelectCompressionNONE
+		if !strings.Contains(objInfo.ContentType, "text/csv") && !strings.Contains(objInfo.ContentType, "application/json") {
 			writeErrorResponse(w, ErrInvalidDataSource, r.URL)
 			return
 		}
@@ -196,28 +199,45 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		writeErrorResponse(w, ErrExpressionTooLong, r.URL)
 		return
 	}
-	if selectReq.InputSerialization.CSV == nil || selectReq.OutputSerialization.CSV == nil {
+	if selectReq.InputSerialization.CSV == nil && selectReq.InputSerialization.JSON == nil {
 		writeErrorResponse(w, ErrInvalidRequestParameter, r.URL)
 		return
 	}
-	if selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoUse &&
-		selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoNone &&
-		selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoIgnore &&
-		selectReq.InputSerialization.CSV.FileHeaderInfo != "" {
-		writeErrorResponse(w, ErrInvalidFileHeaderInfo, r.URL)
-		return
-	}
-	if selectReq.OutputSerialization.CSV.QuoteFields != CSVQuoteFieldsAlways &&
-		selectReq.OutputSerialization.CSV.QuoteFields != CSVQuoteFieldsAsNeeded &&
-		selectReq.OutputSerialization.CSV.QuoteFields != "" {
-		writeErrorResponse(w, ErrInvalidQuoteFields, r.URL)
-		return
-	}
-	if len(selectReq.InputSerialization.CSV.RecordDelimiter) > 2 {
+	if selectReq.OutputSerialization.CSV == nil && selectReq.OutputSerialization.JSON == nil {
 		writeErrorResponse(w, ErrInvalidRequestParameter, r.URL)
 		return
 	}
 
+	if selectReq.InputSerialization.CSV != nil {
+		if selectReq.InputSerialization.CSV.FileHeaderInfo != s3select.CSVFileHeaderInfoUse &&
+			selectReq.InputSerialization.CSV.FileHeaderInfo != s3select.CSVFileHeaderInfoNone &&
+			selectReq.InputSerialization.CSV.FileHeaderInfo != s3select.CSVFileHeaderInfoIgnore &&
+			selectReq.InputSerialization.CSV.FileHeaderInfo != "" {
+			writeErrorResponse(w, ErrInvalidFileHeaderInfo, r.URL)
+			return
+		}
+		if selectReq.OutputSerialization.CSV.QuoteFields != s3select.CSVQuoteFieldsAlways &&
+			selectReq.OutputSerialization.CSV.QuoteFields != s3select.CSVQuoteFieldsAsNeeded &&
+			selectReq.OutputSerialization.CSV.QuoteFields != "" {
+			writeErrorResponse(w, ErrInvalidQuoteFields, r.URL)
+			return
+		}
+		if len(selectReq.InputSerialization.CSV.RecordDelimiter) > 2 {
+			writeErrorResponse(w, ErrInvalidRequestParameter, r.URL)
+			return
+		}
+
+	}
+	if selectReq.InputSerialization.JSON != nil {
+		if selectReq.InputSerialization.JSON.Type != s3select.JSONTypeDocument &&
+			selectReq.InputSerialization.JSON.Type != s3select.JSONLinesType &&
+			selectReq.InputSerialization.JSON.Type != "" {
+			writeErrorResponse(w, ErrInvalidJSONType, r.URL)
+			return
+		}
+
+	}
+
 	// Set encryption response headers
 	if objectAPI.IsEncryptionSupported() {
 		if crypto.IsEncrypted(objInfo.UserDefined) {
@@ -231,45 +251,30 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		}
 	}
 
-	//s3select //Options
-	if selectReq.OutputSerialization.CSV.FieldDelimiter == "" {
-		selectReq.OutputSerialization.CSV.FieldDelimiter = ","
+	s3s, err := s3select.New(gr, objInfo.Size, selectReq)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
 	}
-	if selectReq.InputSerialization.CSV.FileHeaderInfo == "" {
-		selectReq.InputSerialization.CSV.FileHeaderInfo = CSVFileHeaderInfoNone
+
+	// Parses the select query and checks for an error
+	_, _, _, _, _, _, err = s3select.ParseSelect(s3s)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
 	}
-	if selectReq.InputSerialization.CSV.RecordDelimiter == "" {
-		selectReq.InputSerialization.CSV.RecordDelimiter = "\n"
+
+	// Executes the query on data-set
+	if err = s3select.Execute(w, s3s); err != nil {
+		logger.LogIf(ctx, err)
+
 	}
-	if selectReq.InputSerialization.CSV != nil {
-		options := &s3select.Options{
-			HasHeader:            selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoNone,
-			RecordDelimiter:      selectReq.InputSerialization.CSV.RecordDelimiter,
-			FieldDelimiter:       selectReq.InputSerialization.CSV.FieldDelimiter,
-			Comments:             selectReq.InputSerialization.CSV.Comments,
-			Name:                 "S3Object", // Default table name for all objects
-			ReadFrom:             gr,
-			Compressed:           string(selectReq.InputSerialization.CompressionType),
-			Expression:           selectReq.Expression,
-			OutputFieldDelimiter: selectReq.OutputSerialization.CSV.FieldDelimiter,
-			StreamSize:           objInfo.Size,
-			HeaderOpt:            selectReq.InputSerialization.CSV.FileHeaderInfo == CSVFileHeaderInfoUse,
-			Progress:             selectReq.RequestProgress.Enabled,
-		}
-		s3s, err := s3select.NewInput(options)
-		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-			return
-		}
-		_, _, _, _, _, _, err = s3s.ParseSelect(selectReq.Expression)
-		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-			return
-		}
-		if err = s3s.Execute(w); err != nil {
-			logger.LogIf(ctx, err)
-		}
+
+	for k, v := range objInfo.UserDefined {
+		logger.GetReqInfo(ctx).SetTags(k, v)
 	}
+
+	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // GetObjectHandler - GET Object
@@ -279,6 +284,8 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObject")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -434,6 +441,12 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		Host:         host,
 		Port:         port,
 	})
+
+	for k, v := range objInfo.UserDefined {
+		logger.GetReqInfo(ctx).SetTags(k, v)
+	}
+
+	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // HeadObjectHandler - HEAD Object
@@ -442,6 +455,8 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadObject")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponseHeadersOnly(w, ErrServerNotInitialized)
@@ -531,6 +546,11 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 			case crypto.S3.IsEncrypted(objInfo.UserDefined):
 				w.Header().Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 			case crypto.SSEC.IsEncrypted(objInfo.UserDefined):
+				// Validate the SSE-C Key set in the header.
+				if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil {
+					writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
+					return
+				}
 				w.Header().Set(crypto.SSECAlgorithm, r.Header.Get(crypto.SSECAlgorithm))
 				w.Header().Set(crypto.SSECKeyMD5, r.Header.Get(crypto.SSECKeyMD5))
 			}
@@ -544,7 +564,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// Set standard object headers.
 	if hErr := setObjectHeaders(w, objInfo, rs); hErr != nil {
-		writeErrorResponse(w, toAPIErrorCode(hErr), r.URL)
+		writeErrorResponseHeadersOnly(w, toAPIErrorCode(hErr))
 		return
 	}
 
@@ -575,6 +595,12 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		Host:         host,
 		Port:         port,
 	})
+
+	for k, v := range objInfo.UserDefined {
+		logger.GetReqInfo(ctx).SetTags(k, v)
+	}
+
+	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // Extract metadata relevant for an CopyObject operation based on conditional
@@ -615,12 +641,14 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
 func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObject")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -735,9 +763,27 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	var reader io.Reader
 	var length = srcInfo.Size
+
+	// Set the actual size to the decrypted size if encrypted.
+	actualSize := srcInfo.Size
+	if crypto.IsEncrypted(srcInfo.UserDefined) {
+		actualSize, err = srcInfo.DecryptedSize()
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+	}
+
 	// No need to compress for remote etcd calls
 	// Pass the decompressed stream to such calls.
-	if srcInfo.IsCompressed() && !isRemoteCallRequired(ctx, srcBucket, dstBucket, objectAPI) {
+	isCompressed := objectAPI.IsCompressionSupported() && isCompressible(r.Header, srcObject) && !isRemoteCallRequired(ctx, srcBucket, dstBucket, objectAPI)
+	if isCompressed {
+		// Storing the compression metadata.
+		srcInfo.UserDefined[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
+		srcInfo.UserDefined[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
+		// Remove all source encrypted related metadata to
+		// avoid copying them in target object.
+		crypto.RemoveInternalEntries(srcInfo.UserDefined)
 		// Open a pipe for compression.
 		// Where pipeWriter is piped to srcInfo.Reader.
 		// gr writes to pipeWriter.
@@ -760,27 +806,35 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		reader = gr
 	}
 
-	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", srcInfo.Size)
+	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualSize)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// Save the original size for later use when we want to copy
-	// encrypted file into an unencrypted one.
-	size := srcInfo.Size
-
 	var encMetadata = make(map[string]string)
-	if objectAPI.IsEncryptionSupported() && !srcInfo.IsCompressed() {
+	if objectAPI.IsEncryptionSupported() && !isCompressed {
+		// Encryption parameters not applicable for this object.
+		if !crypto.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header) {
+			writeErrorResponse(w, toAPIErrorCode(errInvalidEncryptionParameters), r.URL)
+			return
+		}
+		// Encryption parameters not present for this object.
+		if crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && !crypto.SSECopy.IsRequested(r.Header) {
+			writeErrorResponse(w, ErrInvalidSSECustomerAlgorithm, r.URL)
+			return
+		}
 		var oldKey, newKey []byte
 		sseCopyS3 := crypto.S3.IsEncrypted(srcInfo.UserDefined)
-		sseCopyC := crypto.SSECopy.IsRequested(r.Header)
+		sseCopyC := crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header)
 		sseC := crypto.SSEC.IsRequested(r.Header)
 		sseS3 := crypto.S3.IsRequested(r.Header)
-		if sseC || sseS3 {
-			if sseC {
-				newKey, err = ParseSSECustomerRequest(r)
-			}
+
+		isSourceEncrypted := sseCopyC || sseCopyS3
+		isTargetEncrypted := sseC || sseS3
+
+		if sseC {
+			newKey, err = ParseSSECustomerRequest(r)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -807,42 +861,43 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			// Since we are rotating the keys, make sure to update the metadata.
 			srcInfo.metadataOnly = true
 		} else {
-			if sseCopyC || sseCopyS3 {
+			if isSourceEncrypted || isTargetEncrypted {
 				// We are not only copying just metadata instead
 				// we are creating a new object at this point, even
 				// if source and destination are same objects.
 				srcInfo.metadataOnly = false
-				if sseC || sseS3 {
-					size = srcInfo.Size
-				}
 			}
-			if sseC || sseS3 {
+
+			// Calculate the size of the target object
+			var targetSize int64
+
+			switch {
+			case !isSourceEncrypted && !isTargetEncrypted:
+				fallthrough
+			case isSourceEncrypted && isTargetEncrypted:
+				targetSize = srcInfo.Size
+			// Source not encrypted and target encrypted
+			case !isSourceEncrypted && isTargetEncrypted:
+				targetSize = srcInfo.EncryptedSize()
+			case isSourceEncrypted && !isTargetEncrypted:
+				targetSize, _ = srcInfo.DecryptedSize()
+			}
+
+			if isTargetEncrypted {
 				reader, err = newEncryptReader(reader, newKey, dstBucket, dstObject, encMetadata, sseS3)
 				if err != nil {
 					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 					return
 				}
-				// We are not only copying just metadata instead
-				// we are creating a new object at this point, even
-				// if source and destination are same objects.
-				srcInfo.metadataOnly = false
-				if !sseCopyC && !sseCopyS3 {
-					size = srcInfo.EncryptedSize()
-				}
-			} else {
-				if sseCopyC || sseCopyS3 {
-					size, _ = srcInfo.DecryptedSize()
-					delete(srcInfo.UserDefined, crypto.SSEIV)
-					delete(srcInfo.UserDefined, crypto.SSESealAlgorithm)
-					delete(srcInfo.UserDefined, crypto.SSECSealedKey)
-					delete(srcInfo.UserDefined, crypto.SSEMultipart)
-					delete(srcInfo.UserDefined, crypto.S3SealedKey)
-					delete(srcInfo.UserDefined, crypto.S3KMSSealedKey)
-					delete(srcInfo.UserDefined, crypto.S3KMSKeyID)
-				}
 			}
 
-			srcInfo.Reader, err = hash.NewReader(reader, size, "", "", size) // do not try to verify encrypted content
+			if isSourceEncrypted {
+				// Remove all source encrypted related metadata to
+				// avoid copying them in target object.
+				crypto.RemoveInternalEntries(srcInfo.UserDefined)
+			}
+
+			srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", targetSize) // do not try to verify encrypted content
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -934,8 +989,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		host, port = "", ""
 	}
 
-	if srcInfo.IsCompressed() {
-		objInfo.Size = srcInfo.GetActualSize()
+	if objInfo.IsCompressed() {
+		objInfo.Size = actualSize
 	}
 
 	// Notify object created event.
@@ -948,6 +1003,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		Host:       host,
 		Port:       port,
 	})
+
+	for k, v := range objInfo.UserDefined {
+		logger.GetReqInfo(ctx).SetTags(k, v)
+	}
+
+	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // PutObjectHandler - PUT Object
@@ -961,12 +1022,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObject")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -1057,22 +1120,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		putObject = objectAPI.PutObject
 	)
 	reader = r.Body
-	switch rAuthType {
-	default:
-		// For all unknown auth types return error.
-		writeErrorResponse(w, ErrAccessDenied, r.URL)
+
+	// Check if put is allowed
+	if s3Err = isPutAllowed(rAuthType, bucket, object, r); s3Err != ErrNone {
+		writeErrorResponse(w, s3Err, r.URL)
 		return
-	case authTypeAnonymous:
-		if !globalPolicySys.IsAllowed(policy.Args{
-			Action:          policy.PutObjectAction,
-			BucketName:      bucket,
-			ConditionValues: getConditionValues(r, ""),
-			IsOwner:         false,
-			ObjectName:      object,
-		}) {
-			writeErrorResponse(w, ErrAccessDenied, r.URL)
-			return
-		}
+	}
+
+	switch rAuthType {
 	case authTypeStreamingSigned:
 		// Initialize stream signature verifier.
 		reader, s3Err = newSignV4ChunkedReader(r)
@@ -1119,7 +1174,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			_, cerr := io.CopyN(snappyWriter, actualReader, actualSize)
 			snappyWriter.Close()
 			pipeWriter.CloseWithError(cerr)
-
 		}()
 
 		// Set compression metrics.
@@ -1209,6 +1263,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		Host:       host,
 		Port:       port,
 	})
+
+	for k, v := range objInfo.UserDefined {
+		logger.GetReqInfo(ctx).SetTags(k, v)
+	}
+
+	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 /// Multipart objectAPIHandlers
@@ -1222,12 +1282,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "NewMultipartUpload")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -1314,12 +1376,14 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObjectPart")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -1400,11 +1464,17 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	defer gr.Close()
 	srcInfo := gr.ObjInfo
 
-	var actualPartSize int64
-	actualPartSize = srcInfo.Size
+	actualPartSize := srcInfo.Size
+	if crypto.IsEncrypted(srcInfo.UserDefined) {
+		actualPartSize, err = srcInfo.DecryptedSize()
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+	}
 
 	// Special care for CopyObjectPart
-	if partRangeErr := checkCopyPartRangeWithSize(rs, srcInfo.Size); partRangeErr != nil {
+	if partRangeErr := checkCopyPartRangeWithSize(rs, actualPartSize); partRangeErr != nil {
 		writeCopyPartErr(w, partRangeErr, r.URL)
 		return
 	}
@@ -1415,22 +1485,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	}
 
 	// Get the object offset & length
-	startOffset, length, _ := rs.GetOffsetLength(srcInfo.Size)
-
-	if rangeHeader != "" {
-		actualPartSize = length
-	}
-
-	if objectAPI.IsEncryptionSupported() {
-		if crypto.IsEncrypted(srcInfo.UserDefined) {
-			decryptedSize, decryptErr := srcInfo.DecryptedSize()
-			if decryptErr != nil {
-				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-				return
-			}
-			startOffset, length, _ = rs.GetOffsetLength(decryptedSize)
-		}
-	}
+	startOffset, length, _ := rs.GetOffsetLength(actualPartSize)
+	actualPartSize = length
 
 	/// maximum copy size for multipart objects in a single operation
 	if isMaxAllowedPartSize(length) {
@@ -1441,8 +1497,17 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	var reader io.Reader
 	var getLength = length
 
-	// Need to decompress only for range-enabled copy parts.
-	if srcInfo.IsCompressed() && rangeHeader != "" {
+	var li ListPartsInfo
+	li, err = objectAPI.ListObjectParts(ctx, dstBucket, dstObject, uploadID, 0, 1)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+	// Read compression metadata preserved in the init multipart for the decision.
+	_, compressPart := li.UserDefined[ReservedMetadataPrefix+"compression"]
+	isCompressed := compressPart
+	// Compress only if the compression is enabled during initial multipart.
+	if isCompressed {
 		// Open a pipe for compression.
 		// Where pipeWriter is piped to srcInfo.Reader.
 		// gr writes to pipeWriter.
@@ -1468,13 +1533,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	if objectAPI.IsEncryptionSupported() && !srcInfo.IsCompressed() {
-		var li ListPartsInfo
-		li, err = objectAPI.ListObjectParts(ctx, dstBucket, dstObject, uploadID, 0, 1)
-		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-			return
-		}
+	if objectAPI.IsEncryptionSupported() && !isCompressed {
 		if crypto.IsEncrypted(li.UserDefined) {
 			if !hasServerSideEncryptionHeader(r.Header) {
 				writeErrorResponse(w, ErrSSEMultipartEncrypted, r.URL)
@@ -1536,12 +1595,14 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectPart")
 
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) {
 		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
@@ -1611,41 +1672,30 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		md5hex    = hex.EncodeToString(md5Bytes)
 		sha256hex = ""
 		reader    io.Reader
+		s3Error   APIErrorCode
 	)
 	reader = r.Body
 
-	switch rAuthType {
-	default:
-		// For all unknown auth types return error.
-		writeErrorResponse(w, ErrAccessDenied, r.URL)
+	if s3Error = isPutAllowed(rAuthType, bucket, object, r); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
 		return
-	case authTypeAnonymous:
-		if !globalPolicySys.IsAllowed(policy.Args{
-			Action:          policy.PutObjectAction,
-			BucketName:      bucket,
-			ConditionValues: getConditionValues(r, ""),
-			IsOwner:         false,
-			ObjectName:      object,
-		}) {
-			writeErrorResponse(w, ErrAccessDenied, r.URL)
-			return
-		}
+	}
+
+	switch rAuthType {
 	case authTypeStreamingSigned:
 		// Initialize stream signature verifier.
-		var s3Error APIErrorCode
 		reader, s3Error = newSignV4ChunkedReader(r)
 		if s3Error != ErrNone {
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	case authTypeSignedV2, authTypePresignedV2:
-		s3Error := isReqAuthenticatedV2(r)
-		if s3Error != ErrNone {
+		if s3Error = isReqAuthenticatedV2(r); s3Error != ErrNone {
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error := reqSignatureV4Verify(r, globalServerConfig.GetRegion()); s3Error != ErrNone {
+		if s3Error = reqSignatureV4Verify(r, globalServerConfig.GetRegion()); s3Error != ErrNone {
 			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
@@ -1710,6 +1760,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		}
 	}
 
+	isEncrypted := false
 	if objectAPI.IsEncryptionSupported() && !isCompressed {
 		var li ListPartsInfo
 		li, err = objectAPI.ListObjectParts(ctx, bucket, object, uploadID, 0, 1)
@@ -1718,7 +1769,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			return
 		}
 		if crypto.IsEncrypted(li.UserDefined) {
-			if !hasServerSideEncryptionHeader(r.Header) {
+			isEncrypted = true
+			if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(li.UserDefined) {
 				writeErrorResponse(w, ErrSSEMultipartEncrypted, r.URL)
 				return
 			}
@@ -1746,7 +1798,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			mac.Write(partIDbin[:])
 			partEncryptionKey := mac.Sum(nil)
 
-			reader, err = sio.EncryptReader(reader, sio.Config{Key: partEncryptionKey})
+			reader, err = sio.EncryptReader(hashReader, sio.Config{Key: partEncryptionKey})
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -1762,7 +1814,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	}
 
 	putObjectPart := objectAPI.PutObjectPart
-	if api.CacheAPI() != nil && !hasServerSideEncryptionHeader(r.Header) {
+	if api.CacheAPI() != nil && !isEncrypted {
 		putObjectPart = api.CacheAPI().PutObjectPart
 	}
 	partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, hashReader, opts)
@@ -1787,6 +1839,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AbortMultipartUpload")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -1814,7 +1868,11 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 		}
 	}
 
-	uploadID, _, _, _ := getObjectResources(r.URL.Query())
+	uploadID, _, _, _, s3Error := getObjectResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 	if err := abortMultipartUpload(ctx, bucket, object, uploadID); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -1826,6 +1884,8 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectParts")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -1841,7 +1901,11 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	uploadID, partNumberMarker, maxParts, _ := getObjectResources(r.URL.Query())
+	uploadID, partNumberMarker, maxParts, _, s3Error := getObjectResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 	if partNumberMarker < 0 {
 		writeErrorResponse(w, ErrInvalidPartNumberMarker, r.URL)
 		return
@@ -1866,6 +1930,8 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CompleteMultipartUpload")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -1890,7 +1956,11 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	// Get upload id.
-	uploadID, _, _, _ := getObjectResources(r.URL.Query())
+	uploadID, _, _, _, s3Error := getObjectResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 
 	completeMultipartBytes, err := goioutil.ReadAll(r.Body)
 	if err != nil {
@@ -1972,6 +2042,12 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		Host:       host,
 		Port:       port,
 	})
+
+	for k, v := range objInfo.UserDefined {
+		logger.GetReqInfo(ctx).SetTags(k, v)
+	}
+
+	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 /// Delete objectAPIHandlers
@@ -1980,6 +2056,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteObject")
 
+	defer logger.AuditLog(ctx, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]

cmd/object-handlers_test.go

@@ -2378,14 +2378,16 @@ func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketNam
 			req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, objectName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 
 			if err != nil {
-				t.Fatalf("Failed to create HTTP request for NewMultipart request: <ERROR> %v", err)
+				t.Errorf("Failed to create HTTP request for NewMultipart request: <ERROR> %v", err)
+				return
 			}
 			// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
 			// Call the ServeHTTP to executes the registered handler.
 			apiRouter.ServeHTTP(rec, req)
 			// Assert the response code with the expected status.
 			if rec.Code != http.StatusOK {
-				t.Fatalf("Minio %s:  Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusOK, rec.Code)
+				t.Errorf("Minio %s:  Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusOK, rec.Code)
+				return
 			}
 			// decode the response body.
 			decoder := xml.NewDecoder(rec.Body)
@@ -2393,7 +2395,8 @@ func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketNam
 
 			err = decoder.Decode(multipartResponse)
 			if err != nil {
-				t.Fatalf("Minio %s: Error decoding the recorded response Body", instanceType)
+				t.Errorf("Minio %s: Error decoding the recorded response Body", instanceType)
+				return
 			}
 			// push the obtained upload ID from the response into the array.
 			testUploads.Lock()

cmd/policy.go

@@ -17,10 +17,12 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"net/http"
 	"path"
+	"strings"
 	"sync"
 	"time"
 
@@ -115,7 +117,7 @@ func (sys *PolicySys) refresh(objAPI ObjectLayer) error {
 			logger.Info("Found in-consistent bucket policies, Migrating them for Bucket: (%s)", bucket.Name)
 			config.Version = policy.DefaultVersion
 
-			if err = savePolicyConfig(objAPI, bucket.Name, config); err != nil {
+			if err = savePolicyConfig(context.Background(), objAPI, bucket.Name, config); err != nil {
 				logger.LogIf(context.Background(), err)
 				return err
 			}
@@ -131,25 +133,46 @@ func (sys *PolicySys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	// Load PolicySys once during boot.
-	if err := sys.refresh(objAPI); err != nil {
-		return err
-	}
-
-	// Refresh PolicySys in background.
-	go func() {
-		ticker := time.NewTicker(globalRefreshBucketPolicyInterval)
-		defer ticker.Stop()
-		for {
-			select {
-			case <-globalServiceDoneCh:
-				return
-			case <-ticker.C:
-				sys.refresh(objAPI)
+	defer func() {
+		// Refresh PolicySys in background.
+		go func() {
+			ticker := time.NewTicker(globalRefreshBucketPolicyInterval)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-globalServiceDoneCh:
+					return
+				case <-ticker.C:
+					sys.refresh(objAPI)
+				}
 			}
-		}
+		}()
 	}()
-	return nil
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing policy needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			// Load PolicySys once during boot.
+			if err := sys.refresh(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for policy subsystem to be initialized..")
+					continue
+				}
+				return err
+			}
+			return nil
+		}
+	}
 }
 
 // NewPolicySys - creates new policy system.
@@ -192,7 +215,7 @@ func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, err
 	// Construct path to policy.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
 
-	reader, err := readConfig(context.Background(), objAPI, configFile)
+	configData, err := readConfig(context.Background(), objAPI, configFile)
 	if err != nil {
 		if err == errConfigNotFound {
 			err = BucketPolicyNotFound{Bucket: bucketName}
@@ -201,10 +224,10 @@ func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, err
 		return nil, err
 	}
 
-	return policy.ParseConfig(reader, bucketName)
+	return policy.ParseConfig(bytes.NewReader(configData), bucketName)
 }
 
-func savePolicyConfig(objAPI ObjectLayer, bucketName string, bucketPolicy *policy.Policy) error {
+func savePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, bucketPolicy *policy.Policy) error {
 	data, err := json.Marshal(bucketPolicy)
 	if err != nil {
 		return err
@@ -213,7 +236,7 @@ func savePolicyConfig(objAPI ObjectLayer, bucketName string, bucketPolicy *polic
 	// Construct path to policy.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
 
-	return saveConfig(objAPI, configFile, data)
+	return saveConfig(ctx, objAPI, configFile, data)
 }
 
 func removePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {

cmd/routers.go

@@ -82,6 +82,8 @@ var globalHandlers = []HandlerFunc{
 	// routes them accordingly. Client receives a HTTP error for
 	// invalid/unsupported signatures.
 	setAuthHandler,
+	// Enforce rules specific for TLS requests
+	setSSETLSHandler,
 	// filters HTTP headers which are treated as metadata and are reserved
 	// for internal use only.
 	filterReservedMetadata,
@@ -99,6 +101,9 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
 		registerDistXLRouters(router, endpoints)
 	}
 
+	// Add STS router only enabled if etcd is configured.
+	registerSTSRouter(router)
+
 	// Add Admin RPC router
 	registerAdminRPCRouter(router)
 

cmd/server-main.go

@@ -358,11 +358,17 @@ func serverMain(ctx *cli.Context) {
 		logger.FatalIf(err, "Unable to initialize disk caching")
 	}
 
+	// Create new IAM system.
+	globalIAMSys = NewIAMSys()
+	if err = globalIAMSys.Init(newObject); err != nil {
+		logger.Fatal(err, "Unable to initialize IAM system")
+	}
+
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
 
 	// Initialize policy system.
-	if err := globalPolicySys.Init(newObject); err != nil {
+	if err = globalPolicySys.Init(newObject); err != nil {
 		logger.Fatal(err, "Unable to initialize policy system")
 	}
 
@@ -370,7 +376,7 @@ func serverMain(ctx *cli.Context) {
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 
 	// Initialize notification system.
-	if err := globalNotificationSys.Init(newObject); err != nil {
+	if err = globalNotificationSys.Init(newObject); err != nil {
 		logger.Fatal(err, "Unable to initialize notification system")
 	}
 

cmd/server-startup-msg.go

@@ -116,10 +116,12 @@ func printServerCommonMsg(apiEndpoints []string) {
 
 	// Colorize the message and print.
 	logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
-	logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
-	logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
-	if region != "" {
-		logger.StartupMessage(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
+	if isTerminal() {
+		logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+		if region != "" {
+			logger.StartupMessage(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
+		}
 	}
 	printEventNotifiers()
 
@@ -151,13 +153,15 @@ func printCLIAccessMsg(endPoint string, alias string) {
 	cred := globalServerConfig.GetCredential()
 
 	// Configure 'mc', following block prints platform specific information for minio client.
-	logger.StartupMessage(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
-	if runtime.GOOS == globalWindowsOSName {
-		mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
-		logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-	} else {
-		mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
-		logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+	if isTerminal() {
+		logger.StartupMessage(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
+		if runtime.GOOS == globalWindowsOSName {
+			mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
+			logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+		} else {
+			mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
+			logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+		}
 	}
 }
 

cmd/server_test.go

@@ -1046,7 +1046,8 @@ func (s *TestSuiteCommon) TestPutBucket(c *check) {
 			client := http.Client{Transport: s.transport}
 			response, err := client.Do(request)
 			if err != nil {
-				c.Fatalf("Put bucket Failed: <ERROR> %s", err)
+				c.Errorf("Put bucket Failed: <ERROR> %s", err)
+				return
 			}
 			defer response.Body.Close()
 		}()

cmd/signature-v2.go

@@ -27,6 +27,8 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Signature and API related constants.
@@ -68,7 +70,14 @@ func doesPolicySignatureV2Match(formValues http.Header) APIErrorCode {
 	cred := globalServerConfig.GetCredential()
 	accessKey := formValues.Get("AWSAccessKeyId")
 	if accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 	policy := formValues.Get("Policy")
 	signature := formValues.Get("Signature")
@@ -146,7 +155,14 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 
 	// Validate if access key id same.
 	if accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 
 	// Make sure the request has not expired.
@@ -165,7 +181,7 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 		return ErrInvalidRequest
 	}
 
-	expectedSignature := preSignatureV2(r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires)
+	expectedSignature := preSignatureV2(cred, r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires)
 	if !compareSignatureV2(gotSignature, expectedSignature) {
 		return ErrSignatureDoesNotMatch
 	}
@@ -173,6 +189,29 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 	return ErrNone
 }
 
+func getReqAccessKeyV2(r *http.Request) (string, bool, APIErrorCode) {
+	if accessKey := r.URL.Query().Get("AWSAccessKeyId"); accessKey != "" {
+		owner, s3Err := checkKeyValid(accessKey)
+		return accessKey, owner, s3Err
+	}
+
+	// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
+	// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
+	authFields := strings.Split(r.Header.Get("Authorization"), " ")
+	if len(authFields) != 2 {
+		return "", false, ErrMissingFields
+	}
+
+	// Then will be splitting on ":", this will seprate `AWSAccessKeyId` and `Signature` string.
+	keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":")
+	if len(keySignFields) != 2 {
+		return "", false, ErrMissingFields
+	}
+
+	owner, s3Err := checkKeyValid(keySignFields[0])
+	return keySignFields[0], owner, s3Err
+}
+
 // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
 // Signature = Base64( HMAC-SHA1( YourSecretKey, UTF-8-Encoding-Of( StringToSign ) ) );
 //
@@ -193,41 +232,43 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 //     - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
 // returns true if matches, false otherwise. if error is not nil then it is always false
 
-func validateV2AuthHeader(v2Auth string) APIErrorCode {
+func validateV2AuthHeader(r *http.Request) (auth.Credentials, APIErrorCode) {
+	var cred auth.Credentials
+	v2Auth := r.Header.Get("Authorization")
 	if v2Auth == "" {
-		return ErrAuthHeaderEmpty
+		return cred, ErrAuthHeaderEmpty
 	}
+
 	// Verify if the header algorithm is supported or not.
 	if !strings.HasPrefix(v2Auth, signV2Algorithm) {
-		return ErrSignatureVersionNotSupported
+		return cred, ErrSignatureVersionNotSupported
 	}
 
-	// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
-	// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
-	authFields := strings.Split(v2Auth, " ")
-	if len(authFields) != 2 {
-		return ErrMissingFields
-	}
-
-	// Then will be splitting on ":", this will seprate `AWSAccessKeyId` and `Signature` string.
-	keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":")
-	if len(keySignFields) != 2 {
-		return ErrMissingFields
+	accessKey, owner, apiErr := getReqAccessKeyV2(r)
+	if apiErr != ErrNone {
+		return cred, apiErr
 	}
 
+	cred = globalServerConfig.GetCredential()
 	// Access credentials.
-	cred := globalServerConfig.GetCredential()
-	if keySignFields[0] != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+	if !owner {
+		if globalIAMSys == nil {
+			return cred, ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(accessKey)
+		if !ok {
+			return cred, ErrInvalidAccessKeyID
+		}
 	}
 
-	return ErrNone
+	return cred, ErrNone
 }
 
 func doesSignV2Match(r *http.Request) APIErrorCode {
 	v2Auth := r.Header.Get("Authorization")
-
-	if apiError := validateV2AuthHeader(v2Auth); apiError != ErrNone {
+	cred, apiError := validateV2AuthHeader(r)
+	if apiError != ErrNone {
 		return apiError
 	}
 
@@ -249,12 +290,12 @@ func doesSignV2Match(r *http.Request) APIErrorCode {
 		return ErrInvalidRequest
 	}
 
-	prefix := fmt.Sprintf("%s %s:", signV2Algorithm, globalServerConfig.GetCredential().AccessKey)
+	prefix := fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKey)
 	if !strings.HasPrefix(v2Auth, prefix) {
 		return ErrSignatureDoesNotMatch
 	}
 	v2Auth = v2Auth[len(prefix):]
-	expectedAuth := signatureV2(r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header)
+	expectedAuth := signatureV2(cred, r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header)
 	if !compareSignatureV2(v2Auth, expectedAuth) {
 		return ErrSignatureDoesNotMatch
 	}
@@ -268,15 +309,13 @@ func calculateSignatureV2(stringToSign string, secret string) string {
 }
 
 // Return signature-v2 for the presigned request.
-func preSignatureV2(method string, encodedResource string, encodedQuery string, headers http.Header, expires string) string {
-	cred := globalServerConfig.GetCredential()
+func preSignatureV2(cred auth.Credentials, method string, encodedResource string, encodedQuery string, headers http.Header, expires string) string {
 	stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, expires)
 	return calculateSignatureV2(stringToSign, cred.SecretKey)
 }
 
 // Return the signature v2 of a given request.
-func signatureV2(method string, encodedResource string, encodedQuery string, headers http.Header) string {
-	cred := globalServerConfig.GetCredential()
+func signatureV2(cred auth.Credentials, method string, encodedResource string, encodedQuery string, headers http.Header) string {
 	stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, "")
 	signature := calculateSignatureV2(stringToSign, cred.SecretKey)
 	return signature

cmd/signature-v2_test.go

@@ -223,7 +223,12 @@ func TestValidateV2AuthHeader(t *testing.T) {
 	for i, testCase := range testCases {
 		t.Run(fmt.Sprintf("Case %d AuthStr \"%s\".", i+1, testCase.authString), func(t *testing.T) {
 
-			actualErrCode := validateV2AuthHeader(testCase.authString)
+			req := &http.Request{
+				Header: make(http.Header),
+				URL:    &url.URL{},
+			}
+			req.Header.Set("Authorization", testCase.authString)
+			_, actualErrCode := validateV2AuthHeader(req)
 
 			if testCase.expectedError != actualErrCode {
 				t.Errorf("Expected the error code to be %v, got %v.", testCase.expectedError, actualErrCode)

cmd/signature-v4-parser.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"net/http"
 	"net/url"
 	"strings"
 	"time"
@@ -46,6 +47,24 @@ func (c credentialHeader) getScope() string {
 	}, "/")
 }
 
+func getReqAccessKeyV4(r *http.Request, region string) (string, bool, APIErrorCode) {
+	ch, err := parseCredentialHeader("Credential="+r.URL.Query().Get("X-Amz-Credential"), region)
+	if err != ErrNone {
+		// Strip off the Algorithm prefix.
+		v4Auth := strings.TrimPrefix(r.Header.Get("Authorization"), signV4Algorithm)
+		authFields := strings.Split(strings.TrimSpace(v4Auth), ",")
+		if len(authFields) != 3 {
+			return "", false, ErrMissingFields
+		}
+		ch, err = parseCredentialHeader(authFields[0], region)
+		if err != ErrNone {
+			return "", false, err
+		}
+	}
+	owner, s3Err := checkKeyValid(ch.accessKey)
+	return ch.accessKey, owner, s3Err
+}
+
 // parse credentialHeader string into its structured form.
 func parseCredentialHeader(credElement string, region string) (ch credentialHeader, aec APIErrorCode) {
 	creds := strings.Split(strings.TrimSpace(credElement), "=")

cmd/signature-v4-utils.go

@@ -100,6 +100,23 @@ func isValidRegion(reqRegion string, confRegion string) bool {
 	return reqRegion == confRegion
 }
 
+// check if the access key is valid and recognized, additionally
+// also returns if the access key is owner/admin.
+func checkKeyValid(accessKey string) (bool, APIErrorCode) {
+	var owner = true
+	if globalServerConfig.GetCredential().AccessKey != accessKey {
+		if globalIAMSys == nil {
+			return false, ErrInvalidAccessKeyID
+		}
+		// Check if the access key is part of users credentials.
+		if _, ok := globalIAMSys.GetUser(accessKey); !ok {
+			return false, ErrInvalidAccessKeyID
+		}
+		owner = false
+	}
+	return owner, ErrNone
+}
+
 // sumHMAC calculate hmac between two input byte array.
 func sumHMAC(key []byte, data []byte) []byte {
 	hash := hmac.New(sha256.New, key)

cmd/signature-v4.go

@@ -175,7 +175,14 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
 
 	// Verify if the access key id matches.
 	if credHeader.accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(credHeader.accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 
 	// Get signing key.
@@ -211,7 +218,14 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 
 	// Verify if the access key id matches.
 	if pSignValues.Credential.accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(pSignValues.Credential.accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 
 	// Extract all the signed headers along with its values.
@@ -335,7 +349,14 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string) AP
 
 	// Verify if the access key id matches.
 	if signV4Values.Credential.accessKey != cred.AccessKey {
-		return ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(signV4Values.Credential.accessKey)
+		if !ok {
+			return ErrInvalidAccessKeyID
+		}
 	}
 
 	// Extract date, if not present throw error.

cmd/streaming-signature-v4.go

@@ -64,7 +64,7 @@ func getChunkSignature(cred auth.Credentials, seedSignature string, region strin
 //     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
 // returns signature, error otherwise if the signature mismatches or any other
 // error while parsing and validating.
-func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature string, region string, date time.Time, errCode APIErrorCode) {
+func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature string, region string, date time.Time, errCode APIErrorCode) {
 	// Copy request.
 	req := *r
 
@@ -74,7 +74,7 @@ func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature s
 	// Parse signature version '4' header.
 	signV4Values, errCode := parseSignV4(v4Auth, globalServerConfig.GetRegion())
 	if errCode != ErrNone {
-		return "", "", time.Time{}, errCode
+		return cred, "", "", time.Time{}, errCode
 	}
 
 	// Payload streaming.
@@ -82,17 +82,26 @@ func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature s
 
 	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
 	if payload != req.Header.Get("X-Amz-Content-Sha256") {
-		return "", "", time.Time{}, ErrContentSHA256Mismatch
+		return cred, "", "", time.Time{}, ErrContentSHA256Mismatch
 	}
 
 	// Extract all the signed headers along with its values.
 	extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
 	if errCode != ErrNone {
-		return "", "", time.Time{}, errCode
+		return cred, "", "", time.Time{}, errCode
 	}
+
+	cred = globalServerConfig.GetCredential()
 	// Verify if the access key id matches.
 	if signV4Values.Credential.accessKey != cred.AccessKey {
-		return "", "", time.Time{}, ErrInvalidAccessKeyID
+		if globalIAMSys == nil {
+			return cred, "", "", time.Time{}, ErrInvalidAccessKeyID
+		}
+		var ok bool
+		cred, ok = globalIAMSys.GetUser(signV4Values.Credential.accessKey)
+		if !ok {
+			return cred, "", "", time.Time{}, ErrInvalidAccessKeyID
+		}
 	}
 
 	// Verify if region is valid.
@@ -102,14 +111,14 @@ func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature s
 	var dateStr string
 	if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
 		if dateStr = r.Header.Get("Date"); dateStr == "" {
-			return "", "", time.Time{}, ErrMissingDateHeader
+			return cred, "", "", time.Time{}, ErrMissingDateHeader
 		}
 	}
 	// Parse date header.
 	var err error
 	date, err = time.Parse(iso8601Format, dateStr)
 	if err != nil {
-		return "", "", time.Time{}, ErrMalformedDate
+		return cred, "", "", time.Time{}, ErrMalformedDate
 	}
 
 	// Query string.
@@ -129,11 +138,11 @@ func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature s
 
 	// Verify if signature match.
 	if !compareSignatureV4(newSignature, signV4Values.Signature) {
-		return "", "", time.Time{}, ErrSignatureDoesNotMatch
+		return cred, "", "", time.Time{}, ErrSignatureDoesNotMatch
 	}
 
 	// Return caculated signature.
-	return newSignature, region, date, ErrNone
+	return cred, newSignature, region, date, ErrNone
 }
 
 const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB
@@ -151,10 +160,7 @@ var errMalformedEncoding = errors.New("malformed chunked encoding")
 // NewChunkedReader is not needed by normal applications. The http package
 // automatically decodes chunking when reading response bodies.
 func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, APIErrorCode) {
-	// Access credentials.
-	cred := globalServerConfig.GetCredential()
-
-	seedSignature, region, seedDate, errCode := calculateSeedSignature(cred, req)
+	cred, seedSignature, region, seedDate, errCode := calculateSeedSignature(req)
 	if errCode != ErrNone {
 		return nil, errCode
 	}

cmd/sts-errors.go

@@ -0,0 +1,119 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+)
+
+// writeSTSErrorRespone writes error headers
+func writeSTSErrorResponse(w http.ResponseWriter, errorCode STSErrorCode) {
+	stsError := getSTSError(errorCode)
+	// Generate error response.
+	stsErrorResponse := getSTSErrorResponse(stsError)
+	encodedErrorResponse := encodeResponse(stsErrorResponse)
+	writeResponse(w, stsError.HTTPStatusCode, encodedErrorResponse, mimeXML)
+}
+
+// STSError structure
+type STSError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+// STSErrorResponse - error response format
+type STSErrorResponse struct {
+	XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ ErrorResponse" json:"-"`
+	Error   struct {
+		Type    string `xml:"Type"`
+		Code    string `xml:"Code"`
+		Message string `xml:"Message"`
+	} `xml:"Error"`
+	RequestID string `xml:"RequestId"`
+}
+
+// STSErrorCode type of error status.
+type STSErrorCode int
+
+// Error codes, non exhaustive list - http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithSAML.html
+const (
+	ErrSTSNone STSErrorCode = iota
+	ErrSTSMissingParameter
+	ErrSTSInvalidParameterValue
+	ErrSTSClientGrantsExpiredToken
+	ErrSTSInvalidClientGrantsToken
+	ErrSTSMalformedPolicyDocument
+	ErrSTSNotInitialized
+	ErrSTSInternalError
+)
+
+// error code to STSError structure, these fields carry respective
+// descriptions for all the error responses.
+var stsErrCodeResponse = map[STSErrorCode]STSError{
+	ErrSTSMissingParameter: {
+		Code:           "MissingParameter",
+		Description:    "A required parameter for the specified action is not supplied.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSInvalidParameterValue: {
+		Code:           "InvalidParameterValue",
+		Description:    "An invalid or out-of-range value was supplied for the input parameter.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSClientGrantsExpiredToken: {
+		Code:           "ExpiredToken",
+		Description:    "The client grants that was passed is expired or is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSInvalidClientGrantsToken: {
+		Code:           "InvalidClientGrantsToken",
+		Description:    "The client grants token that was passed could not be validated by Minio.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSMalformedPolicyDocument: {
+		Code:           "MalformedPolicyDocument",
+		Description:    "The request was rejected because the policy document was malformed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSTSNotInitialized: {
+		Code:           "STSNotInitialized",
+		Description:    "STS API not initialized, please try again.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSTSInternalError: {
+		Code:           "InternalError",
+		Description:    "We encountered an internal error generating credentials, please try again.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+}
+
+// getSTSError provides STS Error for input STS error code.
+func getSTSError(code STSErrorCode) STSError {
+	return stsErrCodeResponse[code]
+}
+
+// getErrorResponse gets in standard error and resource value and
+// provides a encodable populated response values
+func getSTSErrorResponse(err STSError) STSErrorResponse {
+	errRsp := STSErrorResponse{}
+	errRsp.Error.Code = err.Code
+	errRsp.Error.Message = err.Description
+	errRsp.RequestID = "3L137"
+	return errRsp
+}

cmd/sts-handlers.go

@@ -0,0 +1,177 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/iam/validator"
+)
+
+const (
+	// STS API version.
+	stsAPIVersion = "2011-06-15"
+)
+
+// stsAPIHandlers implements and provides http handlers for AWS STS API.
+type stsAPIHandlers struct{}
+
+// registerSTSRouter - registers AWS STS compatible APIs.
+func registerSTSRouter(router *mux.Router) {
+	// Initialize STS.
+	sts := &stsAPIHandlers{}
+
+	// STS Router
+	stsRouter := router.NewRoute().PathPrefix("/").Subrouter()
+
+	// AssumeRoleWithClientGrants
+	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithClientGrants)).
+		Queries("Action", "AssumeRoleWithClientGrants").
+		Queries("Token", "{Token:.*}")
+
+}
+
+// AssumedRoleUser - The identifiers for the temporary security credentials that
+// the operation returns. Please also see https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumedRoleUser
+type AssumedRoleUser struct {
+	// The ARN of the temporary security credentials that are returned from the
+	// AssumeRole action. For more information about ARNs and how to use them in
+	// policies, see IAM Identifiers (http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html)
+	// in Using IAM.
+	//
+	// Arn is a required field
+	Arn string
+
+	// A unique identifier that contains the role ID and the role session name of
+	// the role that is being assumed. The role ID is generated by AWS when the
+	// role is created.
+	//
+	// AssumedRoleId is a required field
+	AssumedRoleID string `xml:"AssumeRoleId"`
+	// contains filtered or unexported fields
+}
+
+// AssumeRoleWithClientGrantsResponse contains the result of successful AssumeRoleWithClientGrants request.
+type AssumeRoleWithClientGrantsResponse struct {
+	XMLName          xml.Name           `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithClientGrantsResponse" json:"-"`
+	Result           ClientGrantsResult `xml:"AssumeRoleWithClientGrantsResult"`
+	ResponseMetadata struct {
+		RequestID string `xml:"RequestId,omitempty"`
+	} `xml:"ResponseMetadata,omitempty"`
+}
+
+// ClientGrantsResult - Contains the response to a successful AssumeRoleWithClientGrants
+// request, including temporary credentials that can be used to make Minio API requests.
+type ClientGrantsResult struct {
+	// The identifiers for the temporary security credentials that the operation
+	// returns.
+	AssumedRoleUser AssumedRoleUser `xml:",omitempty"`
+
+	// The intended audience (also known as client ID) of the web identity token.
+	// This is traditionally the client identifier issued to the application that
+	// requested the client grants.
+	Audience string `xml:",omitempty"`
+
+	// The temporary security credentials, which include an access key ID, a secret
+	// access key, and a security (or session) token.
+	//
+	// Note: The size of the security token that STS APIs return is not fixed. We
+	// strongly recommend that you make no assumptions about the maximum size. As
+	// of this writing, the typical size is less than 4096 bytes, but that can vary.
+	// Also, future updates to AWS might require larger sizes.
+	Credentials auth.Credentials `xml:",omitempty"`
+
+	// A percentage value that indicates the size of the policy in packed form.
+	// The service rejects any policy with a packed size greater than 100 percent,
+	// which means the policy exceeded the allowed space.
+	PackedPolicySize int `xml:",omitempty"`
+
+	// The issuing authority of the web identity token presented. For OpenID Connect
+	// ID tokens, this contains the value of the iss field. For OAuth 2.0 access tokens,
+	// this contains the value of the ProviderId parameter that was passed in the
+	// AssumeRoleWithClientGrants request.
+	Provider string `xml:",omitempty"`
+
+	// The unique user identifier that is returned by the identity provider.
+	// This identifier is associated with the Token that was submitted
+	// with the AssumeRoleWithClientGrants call. The identifier is typically unique to
+	// the user and the application that acquired the ClientGrantsToken (pairwise identifier).
+	// For OpenID Connect ID tokens, this field contains the value returned by the identity
+	// provider as the token's sub (Subject) claim.
+	SubjectFromToken string `xml:",omitempty"`
+}
+
+// AssumeRoleWithClientGrants - implementation of AWS STS extension API supporting
+// OAuth2.0 client grants.
+//
+// Eg:-
+//    $ curl https://minio:9000/?Action=AssumeRoleWithClientGrants&Token=<jwt>
+func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AssumeRoleWithClientGrants")
+
+	if globalIAMValidators == nil {
+		writeSTSErrorResponse(w, ErrSTSNotInitialized)
+		return
+	}
+
+	// NOTE: this API only accepts JWT tokens.
+	v, err := globalIAMValidators.Get("jwt")
+	if err != nil {
+		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		return
+	}
+
+	vars := mux.Vars(r)
+	m, err := v.Validate(vars["Token"], r.URL.Query().Get("DurationSeconds"))
+	if err != nil {
+		switch err {
+		case validator.ErrTokenExpired:
+			writeSTSErrorResponse(w, ErrSTSClientGrantsExpiredToken)
+		case validator.ErrInvalidDuration:
+			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		default:
+			logger.LogIf(ctx, err)
+			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		}
+		return
+	}
+
+	secret := globalServerConfig.GetCredential().SecretKey
+	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, ErrSTSInternalError)
+		return
+	}
+
+	// Set the newly generated credentials.
+	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred); err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, ErrSTSInternalError)
+		return
+	}
+
+	encodedSuccessResponse := encodeResponse(&AssumeRoleWithClientGrantsResponse{
+		Result: ClientGrantsResult{Credentials: cred},
+	})
+
+	writeSuccessResponseXML(w, encodedSuccessResponse)
+}

cmd/test-utils_test.go

@@ -354,10 +354,16 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
 	globalMinioPort = port
 	globalMinioAddr = getEndpointsLocalAddr(testServer.Disks)
 
-	globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks)
+	globalConfigSys = NewConfigSys()
+
+	globalIAMSys = NewIAMSys()
+	globalIAMSys.Init(objLayer)
 
-	// Create new policy system.
 	globalPolicySys = NewPolicySys()
+	globalPolicySys.Init(objLayer)
+
+	globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks)
+	globalNotificationSys.Init(objLayer)
 
 	return testServer
 }
@@ -465,6 +471,11 @@ func resetGlobalCacheObjectAPI() {
 	globalCacheObjectAPI = nil
 }
 
+// sets globalIAMSys to `nil`.
+func resetGlobalIAMSys() {
+	globalIAMSys = nil
+}
+
 // Resets all the globals used modified in tests.
 // Resetting ensures that the changes made to globals by one test doesn't affect others.
 func resetTestGlobals() {
@@ -488,14 +499,16 @@ func resetTestGlobals() {
 	resetGlobalHealState()
 	//Reset global disk cache flags
 	resetGlobalCacheEnvs()
-	//set globalCacheObjectAPI to nil
+	// Reset globalCacheObjectAPI to nil
 	resetGlobalCacheObjectAPI()
+	// Reset globalIAMSys to `nil`
+	resetGlobalIAMSys()
 }
 
 // Configure the server for the test run.
 func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) {
 	// Initialize server config.
-	if err = newConfig(obj); err != nil {
+	if err = newSrvConfig(obj); err != nil {
 		return err
 	}
 
@@ -1620,11 +1633,13 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro
 		return xl.storageDisks
 	}
 
-	// Create new notification system.
-	globalNotificationSys = NewNotificationSys(globalServerConfig, endpoints)
+	globalConfigSys = NewConfigSys()
+
+	globalIAMSys = NewIAMSys()
+	globalIAMSys.Init(xl)
 
-	// Create new policy system.
 	globalPolicySys = NewPolicySys()
+	globalNotificationSys = NewNotificationSys(globalServerConfig, endpoints)
 
 	return xl, nil
 }
@@ -1894,9 +1909,11 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 	}
 	bucketFS, fsAPIRouter, err := initAPIHandlerTest(objLayer, endpoints)
 	if err != nil {
-		t.Fatalf("Initialzation of API handler tests failed: <ERROR> %s", err)
+		t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)
 	}
 
+	globalIAMSys = NewIAMSys()
+	globalIAMSys.Init(objLayer)
 	// initialize the server and obtain the credentials and root.
 	// credentials are necessary to sign the HTTP request.
 	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {

cmd/typed-errors.go

@@ -76,3 +76,12 @@ var errBucketAlreadyExists = errors.New("Your previous request to create the nam
 
 // error returned for a negative actual size.
 var errInvalidDecompressedSize = errors.New("Invalid Decompressed Size")
+
+// error returned in IAM subsystem when user doesn't exist.
+var errNoSuchUser = errors.New("Specified user does not exist")
+
+// error returned in IAM subsystem when policy doesn't exist.
+var errNoSuchPolicy = errors.New("Specified canned policy does not exist")
+
+// error returned when access is denied.
+var errAccessDenied = errors.New("Do not have enough permissions to access this resource")

cmd/utils.go

@@ -218,18 +218,26 @@ func startProfiler(profilerType, dirPath string) (interface {
 		Stop()
 	}
 
-	// Enable profiler, supported types are [cpu, mem, block].
+	var profilerFileName string
+
+	// Enable profiler and set the name of the file that pkg/pprof
+	// library creates to store profiling data.
 	switch profilerType {
 	case "cpu":
 		profiler = profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "cpu.pprof"
 	case "mem":
 		profiler = profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "mem.pprof"
 	case "block":
 		profiler = profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "block.pprof"
 	case "mutex":
 		profiler = profile.Start(profile.MutexProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "mutex.pprof"
 	case "trace":
 		profiler = profile.Start(profile.TraceProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+		profilerFileName = "trace.out"
 	default:
 		return nil, errors.New("profiler type unknown")
 	}
@@ -237,7 +245,7 @@ func startProfiler(profilerType, dirPath string) (interface {
 	return &profilerWrapper{
 		stopFn: profiler.Stop,
 		pathFn: func() string {
-			return filepath.Join(dirPath, profilerType+".pprof")
+			return filepath.Join(dirPath, profilerFileName)
 		},
 	}, nil
 }
@@ -391,7 +399,7 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
 	reqInfo := &logger.ReqInfo{
 		RequestID:  w.Header().Get(responseRequestIDKey),
 		RemoteHost: handlers.GetSourceIP(r),
-		UserAgent:  r.Header.Get("user-agent"),
+		UserAgent:  r.UserAgent(),
 		API:        api,
 		BucketName: bucket,
 		ObjectName: object,

cmd/web-handlers.go

@@ -47,6 +47,7 @@ import (
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -73,8 +74,9 @@ type ServerInfoRep struct {
 
 // ServerInfo - get server info.
 func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, reply *ServerInfoRep) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 	host, err := os.Hostname()
 	if err != nil {
@@ -95,6 +97,14 @@ func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, rep
 
 	reply.MinioVersion = Version
 	reply.MinioGlobalInfo = getGlobalInfo()
+	// If ENV creds are not set and incoming user is not owner
+	// disable changing credentials.
+	// TODO: fix this in future and allow changing user credentials.
+	v, ok := reply.MinioGlobalInfo["isEnvCreds"].(bool)
+	if ok && !v {
+		reply.MinioGlobalInfo["isEnvCreds"] = !owner
+	}
+
 	reply.MinioMemory = mem
 	reply.MinioPlatform = platform
 	reply.MinioRuntime = goruntime
@@ -114,8 +124,9 @@ func (web *webAPIHandlers) StorageInfo(r *http.Request, args *AuthArgs, reply *S
 	if objectAPI == nil {
 		return toJSONError(errServerNotInitialized)
 	}
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, _, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 	reply.StorageInfo = objectAPI.StorageInfo(context.Background())
 	reply.UIVersion = browser.UIVersion
@@ -133,8 +144,13 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
 	if objectAPI == nil {
 		return toJSONError(errServerNotInitialized)
 	}
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	// TODO: Allow MakeBucket in future.
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	// Check if bucket is a reserved bucket name or invalid.
@@ -181,8 +197,13 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
 	if objectAPI == nil {
 		return toJSONError(errServerNotInitialized)
 	}
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	// TODO: Allow DeleteBucket in future.
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	ctx := context.Background()
@@ -236,10 +257,11 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
 	if web.CacheAPI() != nil {
 		listBuckets = web.CacheAPI().ListBuckets
 	}
-	authErr := webRequestAuthenticate(r)
-	if authErr != nil {
+
+	if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
 		return toJSONError(authErr)
 	}
+
 	// If etcd, dns federation configured list buckets from etcd.
 	if globalDNSConfig != nil {
 		dnsBuckets, err := globalDNSConfig.List()
@@ -305,43 +327,76 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
 	if objectAPI == nil {
 		return toJSONError(errServerNotInitialized)
 	}
+
 	listObjects := objectAPI.ListObjects
 	if web.CacheAPI() != nil {
 		listObjects = web.CacheAPI().ListObjects
 	}
 
-	// Check if anonymous (non-owner) has access to download objects.
-	readable := globalPolicySys.IsAllowed(policy.Args{
-		Action:          policy.GetObjectAction,
-		BucketName:      args.BucketName,
-		ConditionValues: getConditionValues(r, ""),
-		IsOwner:         false,
-		ObjectName:      args.Prefix + "/",
-	})
-	// Check if anonymous (non-owner) has access to upload objects.
-	writable := globalPolicySys.IsAllowed(policy.Args{
-		Action:          policy.PutObjectAction,
-		BucketName:      args.BucketName,
-		ConditionValues: getConditionValues(r, ""),
-		IsOwner:         false,
-		ObjectName:      args.Prefix + "/",
-	})
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		if authErr == errNoAuthToken {
+			// Check if anonymous (non-owner) has access to download objects.
+			readable := globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.GetObjectAction,
+				BucketName:      args.BucketName,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+				ObjectName:      args.Prefix + "/",
+			})
 
-	if authErr := webRequestAuthenticate(r); authErr != nil {
-		if authErr == errAuthentication {
+			// Check if anonymous (non-owner) has access to upload objects.
+			writable := globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.PutObjectAction,
+				BucketName:      args.BucketName,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+				ObjectName:      args.Prefix + "/",
+			})
+
+			reply.Writable = writable
+			if !readable {
+				// Error out if anonymous user (non-owner) has no access to download or upload objects
+				if !writable {
+					return errAccessDenied
+				}
+				// return empty object list if access is write only
+				return nil
+			}
+		} else {
 			return toJSONError(authErr)
 		}
+	}
+
+	// For authenticated users apply IAM policy.
+	if authErr == nil {
+		readable := globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.GetObjectAction),
+			BucketName:      args.BucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         owner,
+			ObjectName:      args.Prefix + "/",
+		})
+
+		writable := globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.PutObjectAction),
+			BucketName:      args.BucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         owner,
+			ObjectName:      args.Prefix + "/",
+		})
 
 		reply.Writable = writable
 		if !readable {
 			// Error out if anonymous user (non-owner) has no access to download or upload objects
 			if !writable {
-				return errAuthentication
+				return errAccessDenied
 			}
 			// return empty object list if access is write only
 			return nil
 		}
-
 	}
 
 	lo, err := listObjects(context.Background(), args.BucketName, args.Prefix, args.Marker, slashSeparator, 1000)
@@ -400,8 +455,10 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs,
 	if web.CacheAPI() != nil {
 		listObjects = web.CacheAPI().ListObjects
 	}
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 
 	if args.BucketName == "" || len(args.Objects) == 0 {
@@ -420,12 +477,34 @@ next:
 				}
 			}
 
+			if !globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     claims.Subject,
+				Action:          iampolicy.Action(policy.DeleteObjectAction),
+				BucketName:      args.BucketName,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         owner,
+				ObjectName:      objectName,
+			}) {
+				return toJSONError(errAccessDenied)
+			}
+
 			if err = deleteObject(nil, objectAPI, web.CacheAPI(), args.BucketName, objectName, r); err != nil {
 				break next
 			}
 			continue
 		}
 
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.DeleteObjectAction),
+			BucketName:      args.BucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         owner,
+			ObjectName:      objectName,
+		}) {
+			return toJSONError(errAccessDenied)
+		}
+
 		// For directories, list the contents recursively and remove.
 		marker := ""
 		for {
@@ -488,8 +567,12 @@ type GenerateAuthReply struct {
 }
 
 func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, reply *GenerateAuthReply) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 	cred, err := auth.GetNewCredentials()
 	if err != nil {
@@ -516,12 +599,14 @@ type SetAuthReply struct {
 
 // SetAuth - Set accessKey and secretKey credentials.
 func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *SetAuthReply) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 
 	// If creds are set through ENV disallow changing credentials.
-	if globalIsEnvCreds || globalWORMEnabled {
+	// TODO: Multi-user credentials also cannot be changed from browser.
+	if globalIsEnvCreds || globalWORMEnabled || !owner {
 		return toJSONError(errChangeCredNotAllowed)
 	}
 
@@ -553,7 +638,10 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
 			reply.PeerErrMsgs[host.String()] = err.Error()
 		}
 	} else {
-		reply.Token = newAuthToken()
+		reply.Token, err = authenticateWeb(creds.AccessKey, creds.SecretKey)
+		if err != nil {
+			return toJSONError(err)
+		}
 		reply.UIVersion = browser.UIVersion
 	}
 
@@ -569,8 +657,12 @@ type GetAuthReply struct {
 
 // GetAuth - return accessKey and secretKey credentials.
 func (web *webAPIHandlers) GetAuth(r *http.Request, args *WebGenericArgs, reply *GetAuthReply) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 	creds := globalServerConfig.GetCredential()
 	reply.AccessKey = creds.AccessKey
@@ -587,11 +679,19 @@ type URLTokenReply struct {
 
 // CreateURLToken creates a URL token (short-lived) for GET requests.
 func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs, reply *URLTokenReply) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
 	}
 
 	creds := globalServerConfig.GetCredential()
+	if !owner {
+		var ok bool
+		creds, ok = globalIAMSys.GetUser(claims.Subject)
+		if !ok {
+			return toJSONError(errInvalidAccessKeyID)
+		}
+	}
 
 	token, err := authenticateURL(creds.AccessKey, creds.SecretKey)
 	if err != nil {
@@ -605,6 +705,10 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs,
 
 // Upload - file upload handler.
 func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "WebUpload")
+
+	defer logger.AuditLog(ctx, r)
+
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		writeWebErrorResponse(w, errServerNotInitialized)
@@ -619,18 +723,34 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	bucket := vars["bucket"]
 	object := vars["object"]
 
-	if authErr := webRequestAuthenticate(r); authErr != nil {
-		if authErr == errAuthentication {
-			writeWebErrorResponse(w, errAuthentication)
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		if authErr == errNoAuthToken {
+			// Check if anonymous (non-owner) has access to upload objects.
+			if !globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.PutObjectAction,
+				BucketName:      bucket,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+				ObjectName:      object,
+			}) {
+				writeWebErrorResponse(w, errAuthentication)
+				return
+			}
+		} else {
+			writeWebErrorResponse(w, authErr)
 			return
 		}
+	}
 
-		// Check if anonymous (non-owner) has access to upload objects.
-		if !globalPolicySys.IsAllowed(policy.Args{
-			Action:          policy.PutObjectAction,
+	// For authenticated users apply IAM policy.
+	if authErr == nil {
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.PutObjectAction),
 			BucketName:      bucket,
 			ConditionValues: getConditionValues(r, ""),
-			IsOwner:         false,
+			IsOwner:         owner,
 			ObjectName:      object,
 		}) {
 			writeWebErrorResponse(w, errAuthentication)
@@ -690,13 +810,13 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	opts := ObjectOptions{}
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
-		if _, err = objectAPI.GetObjectInfo(context.Background(), bucket, object, opts); err == nil {
+		if _, err = objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil {
 			writeWebErrorResponse(w, errMethodNotAllowed)
 			return
 		}
 	}
 
-	objInfo, err := putObject(context.Background(), bucket, object, hashReader, metadata, opts)
+	objInfo, err := putObject(ctx, bucket, object, hashReader, metadata, opts)
 	if err != nil {
 		writeWebErrorResponse(w, err)
 		return
@@ -718,10 +838,20 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		Host:       host,
 		Port:       port,
 	})
+
+	for k, v := range objInfo.UserDefined {
+		logger.GetReqInfo(ctx).SetTags(k, v)
+	}
+
+	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // Download - file download handler.
 func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "WebDownload")
+
+	defer logger.AuditLog(ctx, r)
+
 	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
@@ -734,13 +864,34 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	object := vars["object"]
 	token := r.URL.Query().Get("token")
 
-	if !isAuthTokenValid(token) {
-		// Check if anonymous (non-owner) has access to download objects.
-		if !globalPolicySys.IsAllowed(policy.Args{
-			Action:          policy.GetObjectAction,
+	claims, owner, authErr := webTokenAuthenticate(token)
+	if authErr != nil {
+		if authErr == errNoAuthToken {
+			// Check if anonymous (non-owner) has access to download objects.
+			if !globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.GetObjectAction,
+				BucketName:      bucket,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+				ObjectName:      object,
+			}) {
+				writeWebErrorResponse(w, errAuthentication)
+				return
+			}
+		} else {
+			writeWebErrorResponse(w, authErr)
+			return
+		}
+	}
+
+	// For authenticated users apply IAM policy.
+	if authErr == nil {
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.Subject,
+			Action:          iampolicy.Action(policy.GetObjectAction),
 			BucketName:      bucket,
 			ConditionValues: getConditionValues(r, ""),
-			IsOwner:         false,
+			IsOwner:         owner,
 			ObjectName:      object,
 		}) {
 			writeWebErrorResponse(w, errAuthentication)
@@ -755,7 +906,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 		getObjectInfo = web.CacheAPI().GetObjectInfo
 		getObject = web.CacheAPI().GetObject
 	}
-	objInfo, err := getObjectInfo(context.Background(), bucket, object, opts)
+	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		writeWebErrorResponse(w, err)
 		return
@@ -825,7 +976,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	// Add content disposition.
 	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(object)))
 
-	if err = getObject(context.Background(), bucket, object, 0, -1, httpWriter, "", opts); err != nil {
+	if err = getObject(ctx, bucket, object, 0, -1, httpWriter, "", opts); err != nil {
 		httpWriter.Close()
 		if objInfo.IsCompressed() {
 			wg.Wait()
@@ -861,6 +1012,12 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 		Host:         host,
 		Port:         port,
 	})
+
+	for k, v := range objInfo.UserDefined {
+		logger.GetReqInfo(ctx).SetTags(k, v)
+	}
+
+	logger.GetReqInfo(ctx).SetTags("etag", objInfo.ETag)
 }
 
 // DownloadZipArgs - Argument for downloading a bunch of files as a zip file.
@@ -880,20 +1037,17 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 		host, port = "", ""
 	}
 
+	ctx := newContext(r, w, "WebDownloadZip")
+
+	defer logger.AuditLog(ctx, r)
+
 	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		writeWebErrorResponse(w, errServerNotInitialized)
 		return
 	}
-	getObject := objectAPI.GetObject
-	if web.CacheAPI() != nil {
-		getObject = web.CacheAPI().GetObject
-	}
-	listObjects := objectAPI.ListObjects
-	if web.CacheAPI() != nil {
-		listObjects = web.CacheAPI().ListObjects
-	}
+
 	// Auth is done after reading the body to accommodate for anonymous requests
 	// when bucket policy is enabled.
 	var args DownloadZipArgs
@@ -905,14 +1059,37 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 	}
 
 	token := r.URL.Query().Get("token")
-	if !isAuthTokenValid(token) {
+	claims, owner, authErr := webTokenAuthenticate(token)
+	if authErr != nil {
+		if authErr == errNoAuthToken {
+			for _, object := range args.Objects {
+				// Check if anonymous (non-owner) has access to download objects.
+				if !globalPolicySys.IsAllowed(policy.Args{
+					Action:          policy.GetObjectAction,
+					BucketName:      args.BucketName,
+					ConditionValues: getConditionValues(r, ""),
+					IsOwner:         false,
+					ObjectName:      pathJoin(args.Prefix, object),
+				}) {
+					writeWebErrorResponse(w, errAuthentication)
+					return
+				}
+			}
+		} else {
+			writeWebErrorResponse(w, authErr)
+			return
+		}
+	}
+
+	// For authenticated users apply IAM policy.
+	if authErr == nil {
 		for _, object := range args.Objects {
-			// Check if anonymous (non-owner) has access to download objects.
-			if !globalPolicySys.IsAllowed(policy.Args{
-				Action:          policy.GetObjectAction,
+			if !globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     claims.Subject,
+				Action:          iampolicy.Action(policy.GetObjectAction),
 				BucketName:      args.BucketName,
 				ConditionValues: getConditionValues(r, ""),
-				IsOwner:         false,
+				IsOwner:         owner,
 				ObjectName:      pathJoin(args.Prefix, object),
 			}) {
 				writeWebErrorResponse(w, errAuthentication)
@@ -921,8 +1098,18 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	getObject := objectAPI.GetObject
+	if web.CacheAPI() != nil {
+		getObject = web.CacheAPI().GetObject
+	}
+	listObjects := objectAPI.ListObjects
+	if web.CacheAPI() != nil {
+		listObjects = web.CacheAPI().ListObjects
+	}
+
 	archive := zip.NewWriter(w)
 	defer archive.Close()
+
 	getObjectInfo := objectAPI.GetObjectInfo
 	if web.CacheAPI() != nil {
 		getObjectInfo = web.CacheAPI().GetObjectInfo
@@ -932,7 +1119,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 	for _, object := range args.Objects {
 		// Writes compressed object file to the response.
 		zipit := func(objectName string) error {
-			info, err := getObjectInfo(context.Background(), args.BucketName, objectName, opts)
+			info, err := getObjectInfo(ctx, args.BucketName, objectName, opts)
 			if err != nil {
 				return err
 			}
@@ -1005,7 +1192,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 				}
 			}
 			httpWriter := ioutil.WriteOnClose(writer)
-			if err = getObject(context.Background(), args.BucketName, objectName, 0, length, httpWriter, "", opts); err != nil {
+			if err = getObject(ctx, args.BucketName, objectName, 0, length, httpWriter, "", opts); err != nil {
 				httpWriter.Close()
 				if info.IsCompressed() {
 					// Wait for decompression go-routine to retire.
@@ -1089,8 +1276,12 @@ func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolic
 		return toJSONError(errServerNotInitialized)
 	}
 
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	bucketPolicy, err := objectAPI.GetBucketPolicy(context.Background(), args.BucketName)
@@ -1138,8 +1329,12 @@ func (web *webAPIHandlers) ListAllBucketPolicies(r *http.Request, args *ListAllB
 		return toJSONError(errServerNotInitialized)
 	}
 
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	bucketPolicy, err := objectAPI.GetBucketPolicy(context.Background(), args.BucketName)
@@ -1185,8 +1380,12 @@ func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolic
 		return toJSONError(errServerNotInitialized)
 	}
 
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	_, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	if !owner {
+		return toJSONError(errAccessDenied)
 	}
 
 	policyType := miniogopolicy.BucketPolicy(args.Policy)
@@ -1263,27 +1462,37 @@ type PresignedGetRep struct {
 
 // PresignedGET - returns presigned-Get url.
 func (web *webAPIHandlers) PresignedGet(r *http.Request, args *PresignedGetArgs, reply *PresignedGetRep) error {
-	if !isHTTPRequestValid(r) {
-		return toJSONError(errAuthentication)
+	claims, owner, authErr := webRequestAuthenticate(r)
+	if authErr != nil {
+		return toJSONError(authErr)
+	}
+	var creds auth.Credentials
+	if !owner {
+		var ok bool
+		creds, ok = globalIAMSys.GetUser(claims.Subject)
+		if !ok {
+			return toJSONError(errInvalidAccessKeyID)
+		}
+	} else {
+		creds = globalServerConfig.GetCredential()
 	}
 
+	region := globalServerConfig.GetRegion()
 	if args.BucketName == "" || args.ObjectName == "" {
 		return &json2.Error{
 			Message: "Bucket and Object are mandatory arguments.",
 		}
 	}
+
 	reply.UIVersion = browser.UIVersion
-	reply.URL = presignedGet(args.HostName, args.BucketName, args.ObjectName, args.Expiry)
+	reply.URL = presignedGet(args.HostName, args.BucketName, args.ObjectName, args.Expiry, creds, region)
 	return nil
 }
 
 // Returns presigned url for GET method.
-func presignedGet(host, bucket, object string, expiry int64) string {
-	cred := globalServerConfig.GetCredential()
-	region := globalServerConfig.GetRegion()
-
-	accessKey := cred.AccessKey
-	secretKey := cred.SecretKey
+func presignedGet(host, bucket, object string, expiry int64, creds auth.Credentials, region string) string {
+	accessKey := creds.AccessKey
+	secretKey := creds.SecretKey
 
 	date := UTCNow()
 	dateStr := date.Format(iso8601Format)

cmd/web-handlers_test.go

@@ -343,7 +343,7 @@ func testDeleteBucketWebHandler(obj ObjectLayer, instanceType string, t TestErrH
 		{"minio", false, token, "specified bucket minio does not exist"},
 		{bucketName, false, token, ""},
 		{bucketName, true, token, "Bucket not empty"},
-		{bucketName, false, "", "Authentication failed"},
+		{bucketName, false, "", "JWT token missing"},
 	}
 
 	for _, test := range testCases {
@@ -1294,7 +1294,7 @@ func testWebGetBucketPolicyHandler(obj ObjectLayer, instanceType string, t TestE
 		},
 	}
 
-	if err = savePolicyConfig(obj, bucketName, bucketPolicy); err != nil {
+	if err = savePolicyConfig(context.Background(), obj, bucketName, bucketPolicy); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -1394,7 +1394,7 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t
 		},
 	}
 
-	if err = savePolicyConfig(obj, bucketName, bucketPolicy); err != nil {
+	if err = savePolicyConfig(context.Background(), obj, bucketName, bucketPolicy); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 

cmd/xl-sets.go

@@ -174,7 +174,7 @@ func (s *xlSets) reInitDisks(refFormat *formatXLV3, storageDisks []StorageAPI, f
 // any given sets.
 func (s *xlSets) connectDisksWithQuorum() {
 	var onlineDisks int
-	for onlineDisks < (len(s.endpoints)/2)+1 {
+	for onlineDisks < len(s.endpoints)/2 {
 		for _, endpoint := range s.endpoints {
 			if s.isConnected(endpoint) {
 				continue
@@ -499,7 +499,7 @@ func (s *xlSets) ListObjectsV2(ctx context.Context, bucket, prefix, continuation
 
 // SetBucketPolicy persist the new policy on the bucket.
 func (s *xlSets) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
-	return savePolicyConfig(s, bucket, policy)
+	return savePolicyConfig(ctx, s, bucket, policy)
 }
 
 // GetBucketPolicy will return a policy on a bucket

cmd/xl-v1-bucket.go

@@ -277,7 +277,7 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
 
 // SetBucketPolicy sets policy on bucket
 func (xl xlObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
-	return savePolicyConfig(xl, bucket, policy)
+	return savePolicyConfig(ctx, xl, bucket, policy)
 }
 
 // GetBucketPolicy will get policy on bucket

cmd/xl-v1-metadata.go

@@ -218,6 +218,8 @@ func (m xlMetaV1) ToObjectInfo(bucket, object string) ObjectInfo {
 		ContentEncoding: m.Meta["content-encoding"],
 	}
 
+	objInfo.backendType = BackendErasure
+
 	// Extract etag from metadata.
 	objInfo.ETag = extractETag(m.Meta)
 

cmd/xl-v1-object.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"path"
 	"strconv"
+	"strings"
 	"sync"
 
 	"github.com/minio/minio/cmd/logger"
@@ -447,19 +448,105 @@ func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string, op
 	return info, nil
 }
 
-// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
-func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
-	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllXLMetadata(ctx, xl.getDisks(), bucket, object)
-
-	// get Quorum for this object
-	readQuorum, _, err := objectQuorumFromMeta(ctx, xl, metaArr, errs)
-	if err != nil {
-		return objInfo, err
+func (xl xlObjects) isObjectCorrupted(metaArr []xlMetaV1, errs []error) (validMeta xlMetaV1, ok bool) {
+	// We can consider an object data not reliable
+	// when xl.json is not found in read quorum disks.
+	var notFoundXLJSON int
+	for _, readErr := range errs {
+		if readErr == errFileNotFound {
+			notFoundXLJSON++
+		}
 	}
 
-	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
-		return objInfo, reducedErr
+	for _, m := range metaArr {
+		if !m.IsValid() {
+			continue
+		}
+		validMeta = m
+		break
+	}
+
+	// Return if the object is indeed corrupted.
+	return validMeta, len(xl.getDisks())-notFoundXLJSON < validMeta.Erasure.DataBlocks
+}
+
+const xlCorruptedSuffix = ".CORRUPTED"
+
+// Renames the corrupted object and makes it visible.
+func renameCorruptedObject(ctx context.Context, bucket, object string, validMeta xlMetaV1, disks []StorageAPI, errs []error) {
+	writeQuorum := validMeta.Erasure.DataBlocks + 1
+
+	// Move all existing objects into corrupted suffix.
+	rename(ctx, disks, bucket, object, bucket, object+xlCorruptedSuffix, true, writeQuorum, []error{errFileNotFound})
+
+	tempObj := mustGetUUID()
+
+	// Get all the disks which do not have the file.
+	var cdisks = make([]StorageAPI, len(disks))
+	for i, merr := range errs {
+		if merr == errFileNotFound {
+			cdisks[i] = disks[i]
+		}
+	}
+
+	for _, disk := range cdisks {
+		if disk == nil {
+			continue
+		}
+
+		// Write empty part file on missing disks.
+		disk.AppendFile(minioMetaTmpBucket, pathJoin(tempObj, "part.1"), []byte{})
+
+		// Write algorithm hash for empty part file.
+		alg := validMeta.Erasure.Checksums[0].Algorithm.New()
+		alg.Write([]byte{})
+
+		// Update the checksums and part info.
+		validMeta.Erasure.Checksums[0] = ChecksumInfo{
+			Name:      validMeta.Erasure.Checksums[0].Name,
+			Algorithm: validMeta.Erasure.Checksums[0].Algorithm,
+			Hash:      alg.Sum(nil),
+		}
+		validMeta.Parts[0] = objectPartInfo{
+			Number: 1,
+			Name:   "part.1",
+		}
+
+		// Write the `xl.json` with the newly calculated metadata.
+		writeXLMetadata(ctx, disk, minioMetaTmpBucket, tempObj, validMeta)
+	}
+
+	// Finally rename all the parts into their respective locations.
+	rename(ctx, cdisks, minioMetaTmpBucket, tempObj, bucket, object+xlCorruptedSuffix, true, writeQuorum, []error{errFileNotFound})
+}
+
+// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
+func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
+	disks := xl.getDisks()
+
+	// Read metadata associated with the object from all disks.
+	metaArr, errs := readAllXLMetadata(ctx, disks, bucket, object)
+
+	var readQuorum int
+	// Having read quorum means we have xl.json in at least N/2 disks.
+	if !strings.HasSuffix(object, xlCorruptedSuffix) {
+		if validMeta, ok := xl.isObjectCorrupted(metaArr, errs); ok {
+			renameCorruptedObject(ctx, bucket, object, validMeta, disks, errs)
+			// Return err file not found since we renamed now the corrupted object
+			return objInfo, errFileNotFound
+		}
+
+		// Not a corrupted object, attempt to get readquorum properly.
+		readQuorum, _, err = objectQuorumFromMeta(ctx, xl, metaArr, errs)
+		if err != nil {
+			return objInfo, err
+		}
+
+	} else {
+
+		// If this is a corrupted object, change read quorum to N/2 disks
+		// so it will be visible to users, so they can delete it.
+		readQuorum = len(xl.getDisks()) / 2
 	}
 
 	// List all the file commit ids from parts metadata.
@@ -604,8 +691,9 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 
-		// Rename the successfully written temporary object to final location.
-		if _, err = rename(ctx, xl.getDisks(), minioMetaTmpBucket, tempObj, bucket, object, true, writeQuorum, nil); err != nil {
+		// Rename the successfully written temporary object to final location. Ignore errFileAccessDenied
+		// error because it means that the target object dir exists and we want to be close to S3 specification.
+		if _, err = rename(ctx, xl.getDisks(), minioMetaTmpBucket, tempObj, bucket, object, true, writeQuorum, []error{errFileAccessDenied}); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
 

docs/bucket/notifications/README.md

@@ -409,7 +409,7 @@ This notification target supports two formats: _namespace_ and _access_.
 
 When the _namespace_ format is used, Minio synchronizes objects in the bucket with entries in a hash. For each entry, the key is formatted as "bucketName/objectName" for an object that exists in the bucket, and the value is the JSON-encoded event data about the operation that created/replaced the object in Minio. When objects are updated or deleted, the corresponding entry in the hash is also updated or deleted.
 
-When the _access_ format is used, Minio appends events to a list using [RPUSH](https://redis.io/commands/rpush). Each item in the list is a JSON encoded list with two items, where the first item is a timestamp string, and second item is a JSON object containing evnet data about the operation that happened in the bucket. No entries appended to the list are updated or deleted by Minio in this format.
+When the _access_ format is used, Minio appends events to a list using [RPUSH](https://redis.io/commands/rpush). Each item in the list is a JSON encoded list with two items, where the first item is a timestamp string, and the second item is a JSON object containing event data about the operation that happened in the bucket. No entries appended to the list are updated or deleted by Minio in this format.
 
 The steps below show how to use this notification target in `namespace` and `access` format.
 

docs/config/README.md

@@ -1,9 +1,8 @@
-# Minio Server `config.json` (v27) Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
-
-Minio server stored all its configuration data in `${HOME}/.minio/config.json` file by default.For all releases beyond minio.RELEASE.2017-08-05T00-00-53Z, the configuration data will be migrated to the Minio's backend directory. All configuration changes can be made using [`mc admin config` get/set commands](https://github.com/minio/mc/blob/master/docs/minio-admin-complete-guide.md). Following sections provide detailed explanation of each fields and how to customize them. A complete example of `config.json` is available [here](https://raw.githubusercontent.com/minio/minio/master/docs/config/config.sample.json)
+# Minio Server Config Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
 
 ## Configuration Directory
-The default configuration directory is `${HOME}/.minio`. Till the release `RELEASE.2018-08-02T23-11-36Z` , Minio server configuration file (`config.json`) was stored in the configuration directory. However for releases beyond `RELEASE.2018-08-18T03-49-57Z`, the configuration file (only), has been migrated to the backend storage directory.
+
+The default configuration directory is `${HOME}/.minio`. Till the release `RELEASE.2018-08-02T23-11-36Z`, Minio server configuration file (`config.json`) was stored in the configuration directory. However for releases beyond `RELEASE.2018-08-18T03-49-57Z`, the configuration file (only), has been migrated to the storage back-end (storage back-end is the directory passed to Minio server while starting the server).
 
 You can override the default configuration directory using `--config-dir` command-line option. Please note that this won't have an effect on the `config.json` file as it is always stored on the backend storage, along with data. Minio server generates a new `config.json` with auto-generated access credentials when its started for the first time.
 
@@ -12,6 +11,7 @@ minio server --config-dir /etc/minio /data
 ```
 
 ### Certificate Directory
+
 TLS certificates are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to Minio server with TLS](https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls).
 
 Following is the directory structure for Minio server with TLS certificates.
@@ -25,14 +25,23 @@ $ tree ~/.minio
 │   └── public.crt
 ```
 
-### Configuration Fields
-#### Editing Configuration Fields
-For releases after minio.RELEASE.2017-08-05T00-00-53Z, configuration changes can be done using [mc]((https://docs.minio.io/docs/minio-client-quickstart-guide))
+
+### Accessing configuration file
+
+All configuration changes can be made using [`mc admin config` get/set commands](https://github.com/minio/mc/blob/master/docs/minio-admin-complete-guide.md). Following sections provide brief explanation of fields and how to customize them. A complete example of `config.json` is available [here](https://raw.githubusercontent.com/minio/minio/master/docs/config/config.sample.json)
+
+#### Editing configuration file fields
+
+
+
 ##### Get current configuration for Minio deployment
+
 ```sh
 $ mc admin config get myminio/ > /tmp/myconfig
 ```
+
 ##### Set current configuration for Minio deployment
+
 ```sh
 $ mc admin config set myminio < /tmp/myconfig
 ```
@@ -40,11 +49,13 @@ $ mc admin config set myminio < /tmp/myconfig
 The `mc admin` config API will evolve soon to be able to configure specific fields using get/set commands.
 
 #### Version
+
 |Field|Type|Description|
 |:---|:---|:---|
-|``version``|_string_| `version` determines the configuration file format. Any older version will be automatically be migrated to the latest version upon startup. [DO NOT EDIT THIS FIELD MANUALLY]|
+|``version``|_string_| `version` determines the configuration file format. Any older version will automatically be migrated to the latest version upon startup. [DO NOT EDIT THIS FIELD MANUALLY]|
 
 #### Credential
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``credential``| | Auth credential for object storage and web access.|
@@ -60,6 +71,7 @@ minio server /data
 ```
 
 #### Region
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``region``| _string_ | `region` describes the physical location of the server. By default it is blank. You may override this field with `MINIO_REGION` environment variable. If you are unsure leave it unset.|
@@ -72,6 +84,7 @@ minio server /data
 ```
 
 #### Worm
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``worm``| _string_ | Enable this to turn on Write-Once-Read-Many. By default it is set to `off`. You may override this field with ``MINIO_WORM`` environment variable.|
@@ -84,6 +97,7 @@ minio server /data
 ```
 
 ### Storage Class
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``storageclass``| | Set storage class for configurable data and parity, as per object basis.|
@@ -93,6 +107,7 @@ minio server /data
 By default, parity for objects with standard storage class is set to `N/2`, and parity for objects with reduced redundancy storage class objects is set to `2`. Read more about storage class support in Minio server [here](https://github.com/minio/minio/blob/master/docs/erasure/storage-class/README.md).
 
 ### Cache
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``drives``| _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled|
@@ -101,6 +116,7 @@ By default, parity for objects with standard storage class is set to `N/2`, and
 |``maxuse`` | _int_ | Percentage of disk available to cache |
 
 #### Notify
+
 |Field|Type|Description|
 |:---|:---|:---|
 |``notify``| |Notify enables bucket notification events for lambda computing via the following targets.|
@@ -117,22 +133,27 @@ By default, parity for objects with standard storage class is set to `N/2`, and
 ## Environment only settings
 
 #### Browser
+
 Enable or disable access to web UI. By default it is set to `on`. You may override this field with ``MINIO_BROWSER`` environment variable.
 
 Example:
+
 ```sh
 export MINIO_BROWSER=off
 minio server /data
 ```
 
 ### Domain
-By default, Minio supports path-style requests which look like http://mydomain.com/bucket/object. MINIO_DOMAIN environmental variable is used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the mattched pattern `$1` is used as bucket and the path is used as object. More information on path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html)
 
+By default, Minio supports path-style requests that are of the format http://mydomain.com/bucket/object. MINIO_DOMAIN environment variable is used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the matched pattern `$1` is used as bucket and the path is used as object. More information on path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html)
 Example:
+
 ```sh
 export MINIO_DOMAIN=mydomain.com
 minio server /data
 ```
 
 ## Explore Further
+
 * [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide)
+* [Configure Minio Server with TLS](https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls)

docs/config/config.sample.json

@@ -1,136 +1,189 @@
 {
-    "version": "29",
-    "credential": {
-        "accessKey": "USWUXHGYZQYFYFFIT3RE",
-        "secretKey": "MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03"
-    },
-    "region": "us-east-1",
-    "worm": "off",
-    "storageclass": {
-        "standard": "",
-        "rrs": ""
-    },
-    "cache": {
-        "drives": [],
-        "expiry": 90,
-        "exclude": [],
-        "maxuse": 80
-    },
-    "usage": {
-        "interval": "3h"
-    },
-    "notify": {
-        "amqp": {
-            "1": {
-                "enable": true,
-                "url": "amqp://myuser:mypassword@localhost:5672",
-                "exchange": "bucketevents",
-                "routingKey": "bucketlogs",
-                "exchangeType": "fanout",
-                "deliveryMode": 0,
-                "mandatory": false,
-                "immediate": false,
-                "durable": false,
-                "internal": false,
-                "noWait": false,
-                "autoDeleted": false
-            }
-        },
-        "nats": {
-            "1": {
-                "enable": true,
-                "address": "0.0.0.0:4222",
-                "subject": "bucketevents",
-                "username": "yourusername",
-                "password": "yoursecret",
-                "token": "",
-                "secure": false,
-                "pingInterval": 0,
-                "streaming": {
-                    "enable": false,
-                    "clusterID": "",
-                    "clientID": "",
-                    "async": false,
-                    "maxPubAcksInflight": 0
-                }
-            }
-        },
-        "elasticsearch": {
-            "1": {
-                "enable": true,
-                "format": "namespace",
-                "url": "http://127.0.0.1:9200",
-                "index": "bucketevents"
-            }
-        },
-        "redis": {
-            "1": {
-                "enable": true,
-                "format": "namespace",
-                "address": "127.0.0.1:6379",
-                "password": "yoursecret",
-                "key": "bucketevents"
-            }
-        },
-        "postgresql": {
-            "1": {
-                "enable": true,
-                "format": "namespace",
-                "connectionString": "",
-                "table": "bucketevents",
-                "host": "127.0.0.1",
-                "port": "5432",
-                "user": "postgres",
-                "password": "mypassword",
-                "database": "bucketevents_db"
-            }
-        },
-        "kafka": {
-            "1": {
-                "enable": true,
-                "brokers": ["localhost:9092"],
-                "topic": "bucketevents",
-                "tls" : {
-                    "enable" : true,
-                    "skipVerify" : false,
-                    "clientAuth" : 0
-                },
-                "sasl" : {
-                    "enable" : true,
-                    "username" : "kafka",
-                    "password" : "kafkapasswd"
-                }
-            }
-        },
-        "webhook": {
-            "1": {
-                "enable": true,
-                "endpoint": "http://requestb.in/1525x9n1"
-            }
-        },
-        "mysql": {
-            "1": {
-                "enable": true,
-                "format": "namespace",
-                "dsnString": "",
-                "table": "minio_images",
-                "host": "172.17.0.1",
-                "port": "3306",
-                "user": "root",
-                "password": "password",
-                "database": "miniodb"
-            }
-        },
-        "mqtt": {
-            "1": {
-                "enable": false,
-                "broker": "tcp://localhost:1883",
-                "topic": "minio",
-                "qos": 1,
-                "clientId": "minio",
-                "username": "",
-                "password": ""
-            }
-        }
-    }
+	"version": "31",
+	"credential": {
+		"accessKey": "36J9X8EZI4KEV1G7EHXA",
+		"secretKey": "ECk2uqOoNqvtJIMQ3WYugvmNPL_-zm3WcRqP5vUM",
+		"expiration": "1970-01-01T00:00:00Z",
+		"status": "enabled"
+	},
+	"region": "",
+	"worm": "off",
+	"storageclass": {
+		"standard": "",
+		"rrs": ""
+	},
+	"cache": {
+		"drives": [],
+		"expiry": 90,
+		"maxuse": 80,
+		"exclude": []
+	},
+	"kms": {
+		"vault": {
+			"endpoint": "",
+			"auth": {
+				"type": "",
+				"approle": {
+					"id": "",
+					"secret": ""
+				}
+			},
+			"key-id": {
+				"name": "",
+				"version": 0
+			}
+		}
+	},
+	"notify": {
+		"amqp": {
+			"1": {
+				"enable": false,
+				"url": "",
+				"exchange": "",
+				"routingKey": "",
+				"exchangeType": "",
+				"deliveryMode": 0,
+				"mandatory": false,
+				"immediate": false,
+				"durable": false,
+				"internal": false,
+				"noWait": false,
+				"autoDeleted": false
+			}
+		},
+		"elasticsearch": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"url": "",
+				"index": ""
+			}
+		},
+		"kafka": {
+			"1": {
+				"enable": false,
+				"brokers": null,
+				"topic": "",
+				"tls": {
+					"enable": false,
+					"skipVerify": false,
+					"clientAuth": 0
+				},
+				"sasl": {
+					"enable": false,
+					"username": "",
+					"password": ""
+				}
+			}
+		},
+		"mqtt": {
+			"1": {
+				"enable": false,
+				"broker": "",
+				"topic": "",
+				"qos": 0,
+				"clientId": "",
+				"username": "",
+				"password": "",
+				"reconnectInterval": 0,
+				"keepAliveInterval": 0
+			}
+		},
+		"mysql": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"dsnString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
+		"nats": {
+			"1": {
+				"enable": false,
+				"address": "",
+				"subject": "",
+				"username": "",
+				"password": "",
+				"token": "",
+				"secure": false,
+				"pingInterval": 0,
+				"streaming": {
+					"enable": false,
+					"clusterID": "",
+					"clientID": "",
+					"async": false,
+					"maxPubAcksInflight": 0
+				}
+			}
+		},
+		"postgresql": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"connectionString": "",
+				"table": "",
+				"host": "",
+				"port": "",
+				"user": "",
+				"password": "",
+				"database": ""
+			}
+		},
+		"redis": {
+			"1": {
+				"enable": false,
+				"format": "",
+				"address": "",
+				"password": "",
+				"key": ""
+			}
+		},
+		"webhook": {
+			"1": {
+				"enable": false,
+				"endpoint": ""
+			}
+		}
+	},
+	"logger": {
+		"console": {
+			"enabled": true
+		},
+		"http": {
+			"target1": {
+				"enabled": false,
+				"endpoint": "https://username:password@example.com/api"
+			}
+		}
+	},
+	"compress": {
+		"enabled": false,
+		"extensions": [
+			".txt",
+			".log",
+			".csv",
+			".json"
+		],
+		"mime-types": [
+			"text/csv",
+			"text/plain",
+			"application/json"
+		]
+	},
+	"openid": {
+		"jwks": {
+			"url": null
+		}
+	},
+	"policy": {
+		"opa": {
+			"url": null,
+			"authToken": ""
+		}
+	}
 }

docs/disk-caching/DESIGN.md

@@ -18,7 +18,7 @@ minio server -h
 
   7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/export1 ... /mnt/export24',
      exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension
-     with expiry upto 40 days.
+     with expiry up to 40 days.
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/export{1..24}"
      $ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf"
      $ export MINIO_CACHE_EXPIRY=40
@@ -31,7 +31,7 @@ minio server -h
 - The cache drives are required to be a filesystem mount point with [`atime`](http://kerolasa.github.io/filetimes.html) support to be enabled on the drive. Alternatively writable directories with atime support can be specified in MINIO_CACHE_DRIVES
 - Expiration of each cached entry takes user provided expiry as a hint, and defaults to 90 days if not provided.
 - Garbage collection sweep of the expired cache entries happens whenever cache usage is > 80% of drive capacity, GC continues until sufficient disk space is reclaimed.
-- An object is only cached when drive has sufficient disk space, upto 100 times the size of the object.
+- An object is only cached when drive has sufficient disk space.
 
 ## Behavior
 Disk caching caches objects for both **uploaded** and **downloaded** objects i.e

docs/disk-caching/README.md

@@ -30,7 +30,7 @@ After updating the cache configuration in /tmp/myconfig , use `mc admin config s
 ```sh
 $ mc admin config set myminio < /tmp/myconfig
 ```
-The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, with expiry upto 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup. Cache max usage is restricted to 80% of disk capacity in this example.
+The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, with expiry up to 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup. Cache max usage is restricted to 80% of disk capacity in this example.
 
 ```bash
 export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/cache{1...3}"

docs/distributed/README.md

@@ -43,7 +43,7 @@ To start a distributed Minio instance, you just need to pass drive locations as
 *Note*
 
 - All the nodes running distributed Minio need to have same access key and secret key for the nodes to connect. To achieve this, it is **mandatory** to export access key and secret key as environment variables, `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY`, on all the nodes before executing Minio server command.
-- All the nodes running distributed Minio need to be on homogenous environments i.e same operating system, same number of disks and same interconnects.
+- All the nodes running distributed Minio need to be in a homogeneous environment, i.e. same operating system, same number of disks and same interconnects.
 - `MINIO_DOMAIN` environment variable should be defined and exported if domain is needed to be set.
 - Minio distributed mode requires fresh directories. If required, the drives can be shared with other applications. You can do this by using a sub-directory exclusive to minio. For example, if you have mounted your volume under `/export`, pass `/export/data` as arguments to Minio server.
 - The IP addresses and drive paths below are for demonstration purposes only, you need to replace these with the actual IP addresses and drive paths/folders.
@@ -60,31 +60,6 @@ export MINIO_SECRET_KEY=<SECRET_KEY>
 minio server http://192.168.1.1{1...8}/export1
 ```
 
-#### Windows (experimental)
-
-```cmd
-set MINIO_ACCESS_KEY=<ACCESS_KEY>
-set MINIO_SECRET_KEY=<SECRET_KEY>
-minio.exe server http://192.168.1.1{1...8}/C:/data
-```
-
-
-Example 2: Start distributed Minio instance on 4 nodes with 4 disks (pictured below), by running this command on all the 4 nodes:
-![Distributed Minio, 4 nodes with 4 disks each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_16.jpg?raw=true)
-
-#### GNU/Linux and macOS
-```sh
-export MINIO_ACCESS_KEY=<ACCESS_KEY>
-export MINIO_SECRET_KEY=<SECRET_KEY>
-minio server http://192.168.1.1{1...4}/export{1...4}
-```
-
-#### Windows (experimental)
-```cmd
-set MINIO_ACCESS_KEY=<ACCESS_KEY>
-set MINIO_SECRET_KEY=<SECRET_KEY>
-minio.exe server http://192.168.1.1{1...4}/C:/data{1...4}
-```
 
 __NOTE:__ `{1...n}` shown have 3 dots! Using only 2 dots `{1..4}` will be interpreted by your shell and won't be passed to minio server, affecting the erasure coding order, which may impact performance and high availability. __Always use `{1...n}` (3 dots!) to allow minio server to optimally erasure-code data__
 
@@ -92,7 +67,7 @@ __NOTE:__ `{1...n}` shown have 3 dots! Using only 2 dots `{1..4}` will be interp
 To test this setup, access the Minio server via browser or [`mc`](https://docs.minio.io/docs/minio-client-quickstart-guide).
 
 ## Explore Further
-- [Minio Large Bucket Suppport Guide](https://docs.minio.io/docs/minio-large-bucket-support-quickstart-guide)
+- [Minio Large Bucket Support Guide](https://docs.minio.io/docs/minio-large-bucket-support-quickstart-guide)
 - [Minio Erasure Code QuickStart Guide](https://docs.minio.io/docs/minio-erasure-code-quickstart-guide)
 - [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
 - [Use `aws-cli` with Minio Server](https://docs.minio.io/docs/aws-cli-with-minio)

docs/docker/README.md

@@ -74,7 +74,7 @@ docker service create --name="minio-service" --secret="access_key" --secret="sec
 Read more about `docker service` [here](https://docs.docker.com/engine/swarm/how-swarm-mode-works/services/)
 
 #### Minio Custom Access and Secret Key files
-To use other secret names follow the instuctions above and replace `access_key` and `secret_key` with your custom names (e.g. `my_secret_key`,`my_custom_key`). Run your service with
+To use other secret names follow the instructions above and replace `access_key` and `secret_key` with your custom names (e.g. `my_secret_key`,`my_custom_key`). Run your service with
 ```
 docker service create --name="minio-service" \
   --secret="my_access_key" \

docs/gateway/README.md

@@ -4,6 +4,6 @@ Minio Gateway adds Amazon S3 compatibility to third party cloud storage provider
 - [Microsoft Azure Blob Storage](https://github.com/minio/minio/blob/master/docs/gateway/azure.md)
 - [Google Cloud Storage](https://github.com/minio/minio/blob/master/docs/gateway/gcs.md)
 - [Backblaze B2](https://github.com/minio/minio/blob/master/docs/gateway/b2.md)
+- [S3](https://github.com/minio/minio/blob/master/docs/gateway/s3.md)
 - [Sia Decentralized Cloud Storage](https://github.com/minio/minio/blob/master/docs/gateway/sia.md) _Alpha release_
 - [Manta Object Storage](https://github.com/minio/minio/blob/master/docs/gateway/manta.md) _Alpha release_
-

docs/gateway/gcs.md

@@ -8,7 +8,7 @@ Minio GCS Gateway adds Amazon S3 compatibility to Google Cloud Storage.
 3. On the Credentials page, select the __Create credentials__ drop-down, then select __Service account key__.
 4. From the __Service account__ drop-down, select __New service account__
 5. Fill up __Service account name__ and __Service account ID__
-6. For the __Role__, click the selec dropdown to choose __Storage__ -> __Storage Admin__ _(Full control of GCS resources)_
+6. For the __Role__, click the select dropdown to choose __Storage__ -> __Storage Admin__ _(Full control of GCS resources)_
 7. Click the __Create__ button. This will download a credentials file to your desktop. Let's call this credentials.json
 
 Note: Alternate ways to setup *Application Default Credentials* is explained [here](https://developers.google.com/identity/protocols/application-default-credentials)

docs/minio-limits.md

@@ -53,4 +53,4 @@ We found the following APIs to be redundant or less useful outside of AWS S3. If
 - ObjectVersions
 
 ### Object name restrictions on Minio
-Object names that contain characters `^*|\" are unsupported on Windows and other file systems which do not support filenames with these characters.
+Object names that contain characters `^*|\/"` are unsupported on Windows and other file systems which do not support filenames with these characters. Note that this list is not exhaustive, and depends on the maintainers of the filesystem itself.

docs/multi-user/README.md

@@ -0,0 +1,73 @@
+# Minio Multi-user Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+Minio supports multiple long term users in addition to default user created during server startup. New users can be added after server starts up, and server can be configured to deny or allow access to buckets and resources to each of these users. This document explains how to add/remove users and modify their access rights.
+
+
+## Get started
+In this document we will explain in detail on how to configure multiple users.
+
+### 1. Prerequisites
+- Install mc - [Minio Client Quickstart Guide](https://docs.minio.io/docs/minio-client-quickstart-guide.html)
+- Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide)
+
+### 2. Create a new user with canned policy
+Use [`mc admin policies`](https://docs.minio.io/docs/minio-admin-complete-guide.html#policies) to create canned policies. Server provides a default set of canned policies namely `writeonly`, `readonly` and `readwrite` *(these policies apply to all resources on the server)*. These can be overridden by custom policies using `mc admin policies` command.
+
+Create new canned policy file `getonly.json`. This policy enables users to download all objects under `my-bucketname`.
+```json
+cat > getonly.json << EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "s3:GetObject"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:s3:::my-bucketname/*"
+      ],
+      "Sid": ""
+    }
+  ]
+}
+EOF
+```
+
+Create new canned policy by name `getonly` using `getonly.json` policy file.
+```
+mc admin policies add myminio getonly getonly.json
+```
+
+Create a new user `newuser` on Minio use `mc admin users`, specify `getonly` canned policy for this `newuser`.
+```
+mc admin users add myminio newuser newuser123 getonly
+```
+
+### 3. Disable user
+Disable user `newuser`.
+```
+mc admin users disable myminio newuser
+```
+
+### 4. Remove user
+Remove the user `newuser`.
+```
+mc admin users remove myminio newuser
+```
+
+### 5. Change user policy
+Change the policy for user `newuser` to `putonly` canned policy.
+```
+mc admin users policy myminio newuser putonly
+```
+
+### 5. List all users
+List all enabled and disabled users.
+```
+mc admin users list myminio
+```
+
+## Explore Further
+- [Minio STS Quickstart Guide](https://docs.minio.io/docs/minio-sts-quickstart-guide)
+- [Minio Admin Complete Guide](https://docs.minio.io/docs/minio-admin-complete-guide.html)
+- [The Minio documentation website](https://docs.minio.io)

docs/orchestration/README.md

@@ -14,7 +14,7 @@ The term cloud-native revolves around the idea of applications deployed as micro
 
 While containers provide isolated application execution environment, orchestration platforms allow seamless scaling by helping replicate and manage containers. Minio extends this by adding isolated storage environment for each tenant.
 
-Minio is built ground up on the cloud-native premise. With features like erasure-coding, distributed and shared setup, it focusses only on storage and does it very well. While, it can be scaled by just replicating Minio instances per tenant via an orchestration platform.  
+Minio is built ground up on the cloud-native premise. With features like erasure-coding, distributed and shared setup, it focuses only on storage and does it very well. While, it can be scaled by just replicating Minio instances per tenant via an orchestration platform.  
 
 > In a cloud-native environment, scalability is not a function of the application but the orchestration platform.
 

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
  minio1:
-  image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+  image: minio/minio:RELEASE.2018-10-18T00-28-58Z
   volumes:
    - data1:/data
   ports:
@@ -15,7 +15,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio2:
-  image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+  image: minio/minio:RELEASE.2018-10-18T00-28-58Z
   volumes:
    - data2:/data
   ports:
@@ -25,7 +25,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio3:
-  image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+  image: minio/minio:RELEASE.2018-10-18T00-28-58Z
   volumes:
    - data3:/data
   ports:
@@ -35,7 +35,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio4:
-  image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+  image: minio/minio:RELEASE.2018-10-18T00-28-58Z
   volumes:
    - data4:/data
   ports:

docs/orchestration/docker-swarm/README.md

@@ -34,6 +34,19 @@ echo "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | docker secret create secret_ke
 
 ## 4. Deploy distributed Minio services
 
+The example Minio stack uses 4 Docker volumes, which are created automatically by deploying the stack. We have to make sure that the services in the stack are always (re)started on the same node, where the service is deployed the first time. 
+Otherwise Docker will create a new volume upon restart of the service on another Docker node, which is not in sync with the other volumes and the stack will fail to start healthy. 
+Before deploying the stack, add labels to the Docker nodes where you want the minio services to run:
+
+```
+docker node update --label-add minio1=true <DOCKER-NODE1>
+docker node update --label-add minio2=true <DOCKER-NODE2>
+docker node update --label-add minio3=true <DOCKER-NODE3>
+docker node update --label-add minio4=true <DOCKER-NODE4>
+```
+
+It is possible to run more than one minio service on one Docker Node. Set the labels accordingly.
+
 Download the [Docker Compose file](https://github.com/minio/minio/blob/master/docs/orchestration/docker-swarm/docker-compose-secrets.yaml?raw=true) on your Swarm master. Then execute the command
 
 ```shell

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,8 @@ version: '3.1'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    hostname: minio1
     volumes:
       - minio1-data:/export
     ports:
@@ -14,13 +15,17 @@ services:
         delay: 10s
         max_attempts: 10
         window: 60s
+      placement:
+        constraints:
+          - node.labels.minio1==true
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
     secrets:
       - secret_key
       - access_key
 
   minio2:
-    image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    hostname: minio2
     volumes:
       - minio2-data:/export
     ports:
@@ -32,13 +37,17 @@ services:
         delay: 10s
         max_attempts: 10
         window: 60s
+      placement:
+        constraints:
+          - node.labels.minio2==true
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
     secrets:
       - secret_key
       - access_key
 
   minio3:
-    image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    hostname: minio3
     volumes:
       - minio3-data:/export
     ports:
@@ -50,13 +59,17 @@ services:
         delay: 10s
         max_attempts: 10
         window: 60s
+      placement:
+        constraints:
+          - node.labels.minio3==true
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
     secrets:
       - secret_key
       - access_key
 
   minio4:
-    image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    hostname: minio4
     volumes:
       - minio4-data:/export
     ports:
@@ -68,6 +81,9 @@ services:
         delay: 10s
         max_attempts: 10
         window: 60s
+      placement:
+        constraints:
+          - node.labels.minio4==true
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
     secrets:
       - secret_key

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,8 @@ version: '3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    hostname: minio1
     volumes:
       - minio1-data:/export
     ports:
@@ -17,10 +18,14 @@ services:
         delay: 10s
         max_attempts: 10
         window: 60s
+      placement:
+        constraints:
+          - node.labels.minio1==true
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio2:
-    image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    hostname: minio2
     volumes:
       - minio2-data:/export
     ports:
@@ -35,10 +40,14 @@ services:
         delay: 10s
         max_attempts: 10
         window: 60s
+      placement:
+        constraints:
+          - node.labels.minio2==true
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio3:
-    image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    hostname: minio3
     volumes:
       - minio3-data:/export
     ports:
@@ -53,10 +62,14 @@ services:
         delay: 10s
         max_attempts: 10
         window: 60s
+      placement:
+        constraints:
+          - node.labels.minio3==true
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio4:
-    image: minio/minio:RELEASE.2018-10-05T01-03-03Z
+    image: minio/minio:RELEASE.2018-10-18T00-28-58Z
+    hostname: minio4
     volumes:
       - minio4-data:/export
     ports:
@@ -71,6 +84,9 @@ services:
         delay: 10s
         max_attempts: 10
         window: 60s
+      placement:
+        constraints:
+          - node.labels.minio4==true
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
 volumes:

docs/orchestration/kubernetes-yaml/README.md

@@ -1,603 +0,0 @@
-# Cloud Native Deployment of Minio on Kubernetes [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
-
-## Table of Contents
-
-- [Prerequisites](#prerequisites)
-- [Minio Standalone Server Deployment](#minio-standalone-server-deployment)
-    - [Standalone Quickstart](#standalone-quickstart)
-    - [Create Persistent Volume Claim](#create-persistent-volume-claim)
-    - [Create Deployment](#create-minio-deployment)
-    - [Create LoadBalancer Service](#create-minio-service)
-  - [Update existing Minio Deployment](#update-existing-minio-deployment)
-  - [Resource cleanup](#standalone-resource-cleanup)
-
-- [Minio Distributed Server Deployment](#minio-distributed-server-deployment)
-    - [Distributed Quickstart](#distributed-quickstart)
-    - [Create Minio Headless Service](#create-minio-headless-service)
-    - [Create Minio Statefulset](#create-minio-statefulset)
-    - [Create LoadBalancer Service](#create-minio-service)
-  - [Update existing Minio StatefulSet](#update-existing-minio-statefulset)
-  - [Deploying on cluster nodes with local host path](#deploying-on-cluster-nodes-with-local-host-path)
-  - [Resource cleanup](#distributed-resource-cleanup)
-
-- [Minio GCS Gateway Deployment](#minio-gcs-gateway-deployment)
-    - [GCS Gateway Quickstart](#gcs-gateway-quickstart)
-    - [Create GCS Credentials Secret](#create-gcs-credentials-secret)
-    - [Create Minio GCS Gateway Deployment](#create-minio-gcs-gateway-deployment)
-    - [Create Minio LoadBalancer Service](#create-minio-loadbalancer-service)
-    - [Update Existing Minio GCS Deployment](#update-existing-minio-gcs-deployment)
-  - [Resource cleanup](#gcs-gateway-resource-cleanup) 
-
-## Prerequisites
-
-To run this example, you need Kubernetes version >=1.4 cluster installed and running, and that you have installed the [`kubectl`](https://kubernetes.io/docs/tasks/kubectl/install/) command line tool in your path. Please see the [getting started guides](https://kubernetes.io/docs/getting-started-guides/) for installation instructions for your platform.
-
-## Minio Standalone Server Deployment
-
-The following section describes the process to deploy standalone [Minio](https://minio.io/) server on Kubernetes. The deployment uses the [official Minio Docker image](https://hub.docker.com/r/minio/minio/~/dockerfile/) from Docker Hub.
-
-This section uses following core components of Kubernetes:
-
-- [_Pods_](https://kubernetes.io/docs/user-guide/pods/)
-- [_Services_](https://kubernetes.io/docs/user-guide/services/)
-- [_Deployments_](https://kubernetes.io/docs/user-guide/deployments/)
-- [_Persistent Volume Claims_](https://kubernetes.io/docs/user-guide/persistent-volumes/#persistentvolumeclaims)
-
-### Standalone Quickstart
-
-Run the below commands to get started quickly
-
-```sh
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-standalone-pvc.yaml?raw=true
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-standalone-deployment.yaml?raw=true
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-standalone-service.yaml?raw=true
-```
-
-### Create Persistent Volume Claim
-
-Minio needs persistent storage to store objects. If there is no
-persistent storage, the data stored in Minio instance will be stored in the container file system and will be wiped off as soon as the container restarts.
-
-Create a persistent volume claim (PVC) to request storage for the Minio instance. Kubernetes looks out for PVs matching the PVC request in the cluster and binds it to the PVC automatically.
-
-This is the PVC description.
-
-```sh
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  # This name uniquely identifies the PVC. This is used in deployment.
-  name: minio-pv-claim
-spec:
-  # Read more about access modes here: http://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes
-  accessModes:
-    # The volume is mounted as read-write by a single node
-    - ReadWriteOnce
-  resources:
-    # This is the request for storage. Should be available in the cluster.
-    requests:
-      storage: 10Gi
-```
-
-Create the PersistentVolumeClaim
-
-```sh
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-standalone-pvc.yaml?raw=true
-persistentvolumeclaim "minio-pv-claim" created
-```
-
-### Create Minio Deployment
-
-A deployment encapsulates replica sets and pods — so, if a pod goes down, replication controller makes sure another pod comes up automatically. This way you won’t need to bother about pod failures and will have a stable Minio service available.
-
-This is the deployment description.
-
-```sh
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  # This name uniquely identifies the Deployment
-  name: minio
-spec:
-  strategy:
-    # Specifies the strategy used to replace old Pods by new ones
-    # Refer: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        # This label is used as a selector in Service definition
-        app: minio
-    spec:
-      # Volumes used by this deployment
-      volumes:
-      - name: data
-        # This volume is based on PVC
-        persistentVolumeClaim:
-          # Name of the PVC created earlier
-          claimName: minio-pv-claim
-      containers:
-      - name: minio
-        # Volume mounts for this container
-        volumeMounts:
-        # Volume 'data' is mounted to path '/data'
-        - name: data 
-          mountPath: "/data"
-        # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-10-05T01-03-03Z
-        args:
-        - server
-        - /data
-        env:
-        # Minio access key and secret key
-        - name: MINIO_ACCESS_KEY
-          value: "minio"
-        - name: MINIO_SECRET_KEY
-          value: "minio123"
-        ports:
-        - containerPort: 9000
-        # Readiness probe detects situations when Minio server instance
-        # is not ready to accept traffic. Kubernetes doesn't forward
-        # traffic to the pod till readiness checks fail.
-        readinessProbe:
-          httpGet:
-            path: /minio/health/ready
-            port: 9000
-          initialDelaySeconds: 120
-          periodSeconds: 20
-        # Liveness probe detects situations where Minio server instance
-        # is not working properly and needs restart. Kubernetes automatically
-        # restarts the pods if liveness checks fail.
-        livenessProbe:
-          httpGet:
-            path: /minio/health/live
-            port: 9000
-          initialDelaySeconds: 120
-          periodSeconds: 20
-```
-
-Create the Deployment
-
-```sh
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-standalone-deployment.yaml?raw=true
-deployment "minio-deployment" created
-```
-
-### Create Minio Service
-
-Now that you have a Minio deployment running, you may either want to access it internally (within the cluster) or expose it as a Service onto an external (outside of your cluster, maybe public internet) IP address, depending on your use case. You can achieve this using Services. There are 3 major service types — default type is ClusterIP, which exposes a service to connection from inside the cluster. NodePort and LoadBalancer are two types that expose services to external traffic.
-
-In this example, we expose the Minio Deployment by creating a LoadBalancer service. This is the service description.
-
-```sh
-apiVersion: v1
-kind: Service
-metadata:
-  # This name uniquely identifies the service
-  name: minio-service
-spec:
-  type: LoadBalancer
-  ports:
-    - port: 9000
-      targetPort: 9000
-      protocol: TCP
-  selector:
-    # Looks for labels `app:minio` in the namespace and applies the spec
-    app: minio
-```
-Create the Minio service
-
-```sh
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-standalone-service.yaml?raw=true
-service "minio-service" created
-```
-
-The `LoadBalancer` service takes couple of minutes to launch. To check if the service was created successfully, run the command
-
-```sh
-kubectl get svc minio-service
-NAME            CLUSTER-IP     EXTERNAL-IP       PORT(S)          AGE
-minio-service   10.55.248.23   104.199.249.165   9000:31852/TCP   1m
-```
-
-### Update existing Minio Deployment
-
-You can update an existing Minio deployment to use a newer Minio release. To do this, use the `kubectl set image` command:
-
-```sh
-kubectl set image deployment/minio-deployment minio=<replace-with-new-minio-image>
-```
-
-Kubernetes will restart the deployment to update the image. You will get a message as shown below, on successful update:
-
-```
-deployment "minio-deployment" image updated
-```
-
-### Standalone Resource cleanup
-
-You can cleanup the cluster using
-
-```sh
-kubectl delete deployment minio \
-&&  kubectl delete pvc minio-pv-claim \
-&& kubectl delete svc minio-service
-```
-
-## Minio Distributed Server Deployment
-
-The following document describes the process to deploy [distributed Minio](https://docs.minio.io/docs/distributed-minio-quickstart-guide) server on Kubernetes. This example uses the [official Minio Docker image](https://hub.docker.com/r/minio/minio/~/dockerfile/) from Docker Hub.
-
-This example uses following core components of Kubernetes:
-
-- [_Pods_](https://kubernetes.io/docs/concepts/workloads/pods/pod/)
-- [_Services_](https://kubernetes.io/docs/concepts/services-networking/service/)
-- [_Statefulsets_](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/)
-
-### Distributed Quickstart
-
-Run the below commands to get started quickly
-
-```sh
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-distributed-headless-service.yaml?raw=true
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-distributed-statefulset.yaml?raw=true
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-distributed-service.yaml?raw=true
-```
-
-### Create Minio Headless Service
-
-Headless Service controls the domain within which StatefulSets are created. The domain managed by this Service takes the form: `$(service name).$(namespace).svc.cluster.local` (where “cluster.local” is the cluster domain), and the pods in this domain take the form: `$(pod-name-{i}).$(service name).$(namespace).svc.cluster.local`. This is required to get a DNS resolvable URL for each of the pods created within the Statefulset.
-
-This is the Headless service description.
-
-```sh
-apiVersion: v1
-kind: Service
-metadata:
-  name: minio
-  labels:
-    app: minio
-spec:
-  clusterIP: None
-  ports:
-    - port: 9000
-      name: minio
-  selector:
-    app: minio
-```
-
-Create the Headless Service
-
-```sh
-$ kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-distributed-headless-service.yaml?raw=true
-service "minio" created
-```
-
-### Create Minio Statefulset
-
-A StatefulSet provides a deterministic name and a unique identity to each pod, making it easy to deploy stateful distributed applications. To launch distributed Minio you need to pass drive locations as parameters to the minio server command. Then, you’ll need to run the same command on all the participating pods. StatefulSets offer a perfect way to handle this requirement.
-
-This is the Statefulset description.
-
-```sh
-apiVersion: apps/v1beta1
-kind: StatefulSet
-metadata:
-  # This name uniquely identifies the StatefulSet
-  name: minio
-spec:
-  serviceName: minio
-  replicas: 4
-  selector:
-    matchLabels:
-      app: minio # has to match .spec.template.metadata.labels
-  template:
-    metadata:
-      labels:
-        app: minio # has to match .spec.selector.matchLabels
-    spec:
-      containers:
-      - name: minio
-        env:
-        - name: MINIO_ACCESS_KEY
-          value: "minio"
-        - name: MINIO_SECRET_KEY
-          value: "minio123"
-        image: minio/minio:RELEASE.2018-10-05T01-03-03Z
-        args:
-        - server
-        - http://minio-0.minio.default.svc.cluster.local/data
-        - http://minio-1.minio.default.svc.cluster.local/data
-        - http://minio-2.minio.default.svc.cluster.local/data
-        - http://minio-3.minio.default.svc.cluster.local/data
-        ports:
-        - containerPort: 9000
-        # These volume mounts are persistent. Each pod in the PetSet
-        # gets a volume mounted based on this field.
-        volumeMounts:
-        - name: data
-          mountPath: /data
-        # Liveness probe detects situations where Minio server instance
-        # is not working properly and needs restart. Kubernetes automatically
-        # restarts the pods if liveness checks fail.
-        livenessProbe:
-          httpGet:
-            path: /minio/health/live
-            port: 9000
-          initialDelaySeconds: 120
-          periodSeconds: 20
-  # These are converted to volume claims by the controller
-  # and mounted at the paths mentioned above.
-  volumeClaimTemplates:
-  - metadata:
-      name: data
-    spec:
-      accessModes:
-        - ReadWriteOnce
-      resources:
-        requests:
-          storage: 10Gi
-```
-
-Create the Statefulset
-
-```sh
-$ kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-distributed-statefulset.yaml?raw=true
-statefulset "minio" created
-```
-
-### Create Minio Service
-
-Now that you have a Minio statefulset running, you may either want to access it internally (within the cluster) or expose it as a Service onto an external (outside of your cluster, maybe public internet) IP address, depending on your use case. You can achieve this using Services. There are 3 major service types — default type is ClusterIP, which exposes a service to connection from inside the cluster. NodePort and LoadBalancer are two types that expose services to external traffic.
-
-In this example, we expose the Minio Deployment by creating a LoadBalancer service. This is the service description.
-
-```sh
-apiVersion: v1
-kind: Service
-metadata:
-  name: minio-service
-spec:
-  type: LoadBalancer
-  ports:
-    - port: 9000
-      targetPort: 9000
-      protocol: TCP
-  selector:
-    app: minio
-```
-Create the Minio service
-
-```sh
-$ kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-distributed-service.yaml?raw=true
-service "minio-service" created
-```
-
-The `LoadBalancer` service takes couple of minutes to launch. To check if the service was created successfully, run the command
-
-```sh
-$ kubectl get svc minio-service
-NAME            CLUSTER-IP     EXTERNAL-IP       PORT(S)          AGE
-minio-service   10.55.248.23   104.199.249.165   9000:31852/TCP   1m
-```
-
-### Update existing Minio StatefulSet
-You can update an existing Minio StatefulSet to use a newer Minio release. To do this, use the `kubectl patch statefulset` command:
-
-```sh
-kubectl patch statefulset minio --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value":"<replace-with-new-minio-image>"}]'
-```
-
-On successful update, you should see the output below
-
-```
-statefulset "minio" patched
-```
-
-Then delete all the pods in your StatefulSet one by one as shown below. Kubernetes will restart those pods for you, using the new image. 
-
-```sh
-kubectl delete minio-0
-```
-
-### Resource cleanup
-
-You can cleanup the cluster using
-```sh
-kubectl delete statefulset minio \
-&&  kubectl delete svc minio \
-&& kubectl delete svc minio-service
-```
-
-### Deploying on cluster nodes with local host path
-
-If your cluster does not have a storage solution or PV abstraction, you must explicitly define what nodes you wish to run Minio on, and define a homogeneous path to a local fast block device available on every host.
-
-This must be changed in the example daemonset: [minio-distributed-daemonset.yaml](minio-distributed-daemonset.yaml)
-
-Specifically the hostpath:
-```yaml
-        hostPath:
-          path: /data/minio/
-```
-
-And the list of hosts:
-```yaml
-        - http://hostname1:9000/data/minio
-        - http://hostname2:9000/data/minio
-        - http://hostname3:9000/data/minio
-        - http://hostname4:9000/data/minio
-```
-
-Once deployed, tag the defined host with the `minio-server=true` label:
-
-```bash
-kubectl label node hostname1  -l minio-server=true
-kubectl label node hostname2  -l minio-server=true
-kubectl label node hostname3  -l minio-server=true
-kubectl label node hostname4  -l minio-server=true
-```
-
-## Minio GCS Gateway Deployment
-
-The following section describes the process to deploy [Minio](https://minio.io/) GCS Gateway on Kubernetes. The deployment uses the [official Minio Docker image](https://hub.docker.com/r/minio/minio/~/dockerfile/) from Docker Hub.
-
-This section uses following core components of Kubernetes:
-
-- [_Secrets_](https://kubernetes.io/docs/concepts/configuration/secret/)
-- [_Services_](https://kubernetes.io/docs/user-guide/services/)
-- [_Deployments_](https://kubernetes.io/docs/user-guide/deployments/)
-
-### GCS Gateway Quickstart
-
-Create the Google Cloud Service credentials file using the steps mentioned [here](https://github.com/minio/minio/blob/master/docs/gateway/gcs.md#create-service-account-key-for-gcs-and-get-the-credentials-file). 
-
-Use the path of file generated above to create a Kubernetes `secret`.
-
-```sh
-kubectl create secret generic gcs-credentials --from-file=/path/to/gcloud/credentials/application_default_credentials.json
-```
-
-Then download the `minio-gcs-gateway-deployment.yaml` file
-
-```sh
-wget https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-gcs-gateway-deployment.yaml?raw=true
-```
-
-Update the section `gcp_project_id` with your GCS project ID. Then run
-
-```sh
-kubectl create -f minio-gcs-gateway-deployment.yaml
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-gcs-gateway-service.yaml?raw=true
-```
-
-### Create GCS Credentials Secret
-
-A `secret` is intended to hold sensitive information, such as passwords, OAuth tokens, and ssh keys. Putting this information in a secret is safer and more flexible than putting it verbatim in a pod definition or in a docker image.
-
-Create the Google Cloud Service credentials file using the steps mentioned [here](https://github.com/minio/minio/blob/master/docs/gateway/gcs.md#create-service-account-key-for-gcs-and-get-the-credentials-file). 
-
-Use the path of file generated above to create a Kubernetes `secret`.
-
-```sh
-kubectl create secret generic gcs-credentials --from-file=/path/to/gcloud/credentials/application_default_credentials.json
-```
-
-### Create Minio GCS Gateway Deployment
-
-A deployment encapsulates replica sets and pods — so, if a pod goes down, replication controller makes sure another pod comes up automatically. This way you won’t need to bother about pod failures and will have a stable Minio service available.
-
-Minio Gateway uses GCS as its storage backend and need to use a GCP `projectid` to identify your credentials. Update the section `gcp_project_id` with your
-GCS project ID. This is the deployment description.
-
-```sh
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  # This name uniquely identifies the Deployment
-  name: minio-deployment
-spec:
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        # Label is used as selector in the service.
-        app: minio
-    spec:
-      # Refer to the secret created earlier
-      volumes:
-      - name: gcs-credentials
-        secret:
-          # Name of the Secret created earlier
-          secretName: gcs-credentials
-      containers:
-      - name: minio
-        # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-10-05T01-03-03Z
-        args:
-        - gateway
-        - gcs
-        - gcp_project_id
-        env:
-        # Minio access key and secret key
-        - name: MINIO_ACCESS_KEY
-          value: "minio"
-        - name: MINIO_SECRET_KEY
-          value: "minio123"
-        # Google Cloud Service uses this variable
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          value: "/etc/credentials/application_default_credentials.json"
-        ports:
-        - containerPort: 9000
-        # Mount the volume into the pod
-        volumeMounts:
-        - name: gcs-credentials
-          mountPath: "/etc/credentials"
-          readOnly: true
-```
-
-Create the Deployment
-
-```sh
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-gcs-gateway-deployment.yaml?raw=true
-deployment "minio-deployment" created
-```
-
-### Create Minio LoadBalancer Service
-
-Now that you have a Minio deployment running, you may either want to access it internally (within the cluster) or expose it as a Service onto an external (outside of your cluster, maybe public internet) IP address, depending on your use case. You can achieve this using Services. There are 3 major service types — default type is ClusterIP, which exposes a service to connection from inside the cluster. NodePort and LoadBalancer are two types that expose services to external traffic.
-
-In this example, we expose the Minio Deployment by creating a LoadBalancer service. This is the service description.
-
-```sh
-apiVersion: v1
-kind: Service
-metadata:
-  name: minio-service
-spec:
-  type: LoadBalancer
-  ports:
-    - port: 9000
-      targetPort: 9000
-      protocol: TCP
-  selector:
-    app: minio
-```
-Create the Minio service
-
-```sh
-kubectl create -f https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes-yaml/minio-gcs-gateway-service.yaml?raw=true
-service "minio-service" created
-```
-
-The `LoadBalancer` service takes couple of minutes to launch. To check if the service was created successfully, run the command
-
-```sh
-kubectl get svc minio-service
-NAME            CLUSTER-IP     EXTERNAL-IP       PORT(S)          AGE
-minio-service   10.55.248.23   104.199.249.165   9000:31852/TCP   1m
-```
-
-### Update Existing Minio GCS Deployment
-
-You can update an existing Minio deployment to use a newer Minio release. To do this, use the `kubectl set image` command:
-
-```sh
-kubectl set image deployment/minio-deployment minio=<replace-with-new-minio-image>
-```
-
-Kubernetes will restart the deployment to update the image. You will get a message as shown below, on successful update:
-
-```
-deployment "minio-deployment" image updated
-```
-
-### GCS Gateway Resource Cleanup
-
-You can cleanup the cluster using
-
-```sh
-kubectl delete deployment minio-deployment \
-&&  kubectl delete secret gcs-credentials 
-```