Change storageRESTTimeout to 1minute (#7398)

In scenario 1

```
- bucket/object-prefix
- bucket/object-prefix/object
```

Server responds with `XMinioParentIsObject`

In scenario 2

```
- bucket/object-prefix/object
- bucket/object-prefix
```

Server responds with `XMinioObjectExistsAsDirectory`

Fixes #6566

.travis.yml

@@ -17,6 +17,7 @@ matrix:
       env:
         - ARCH=x86_64
         - CGO_ENABLED=0
+        - GO111MODULE=on
       go: 1.11.4
       script:
         - make
@@ -44,7 +45,7 @@ before_script:
   - if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
 
 before_install:
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install 11.10.1 ; fi
 
 after_success:
 - bash <(curl -s https://codecov.io/bash)

Dockerfile.simpleci

@@ -7,12 +7,39 @@ COPY . /go/src/github.com/minio/minio
 WORKDIR /go/src/github.com/minio/minio
 
 RUN apt-get update && apt-get install -y jq
+ENV GO111MODULE=on
+
+RUN git config --global http.cookiefile /gitcookie/.gitcookie
+
 RUN make
 RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
 RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
+
+RUN apt-get update && \
+      apt-get -y install sudo
+RUN touch  /etc/sudoers
+
+RUN echo "root ALL=(ALL) ALL" >> /etc/sudoers
+RUN echo "ci ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+RUN echo "Defaults    env_reset" >> /etc/sudoers
+RUN echo 'Defaults    secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go:/usr/local/go/bin"' >> /etc/sudoers
+
+RUN mkdir -p /home/ci/.cache
+
+RUN groupadd -g 999 ci && \
+    useradd -r -u 999 -g ci ci
+
+RUN chown -R ci:ci /go
+RUN chown -R ci:ci /home/ci
+RUN chmod -R a+rw /go 
+
+USER ci
+
 RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done
+RUN make verifiers
+RUN make crosscompile
+RUN make coverage
 RUN make verify
-RUN make coverage || true
 
 #-------------------------------------------------------------
 # Stage 2: Test Frontend
@@ -35,9 +62,6 @@ COPY --from=0 /go/src/github.com/minio/minio/minio ./minio
 COPY buildscripts/gateway-tests.sh ./gateway-tests.sh
 RUN apt-get update && apt-get install -y git wget jq curl dnsmasq
 
-RUN wget https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz && \
-	tar -C /usr/local -xzf go1.11.4.linux-amd64.tar.gz
-
 ENV DEBIAN_FRONTEND noninteractive
 ENV LANG C.UTF-8
 ENV GOROOT /usr/local/go

Makefile

@@ -13,9 +13,10 @@ checks:
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 getdeps:
-	@echo "Installing golint" && go get -u golang.org/x/lint/golint
-	@echo "Installing staticcheck" && go get -u honnef.co/go/tools/...
-	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
+	@mkdir -p $(GOPATH)/bin
+	@echo "Installing golint" && which golint || go get -u golang.org/x/lint/golint
+	@echo "Installing staticcheck" && which staticcheck || wget --quiet -O $(GOPATH)/bin/staticcheck https://github.com/dominikh/go-tools/releases/download/2019.1/staticcheck_linux_amd64 && chmod +x $(GOPATH)/bin/staticcheck
+	@echo "Installing misspell" && which misspell || wget --quiet https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_linux_64bit.tar.gz && tar xvf misspell_0.3.4_linux_64bit.tar.gz && mv misspell $(GOPATH)/bin/misspell && chmod +x $(GOPATH)/bin/misspell && rm -r misspell_0.3.4_linux_64bit.tar.gz
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)

browser/app/js/objects/ObjectActions.js

@@ -67,7 +67,7 @@ export class ObjectActions extends React.Component {
             className="fiad-action"
             onClick={this.shareObject.bind(this)}
           >
-            <i className="fa fa-copy" />
+            <i className="fa fa-share-alt" />
           </a>
           <a
             href=""

buildscripts/gateway-tests.sh

@@ -42,10 +42,9 @@ cd /mint
 export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint}
 export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core"
 export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security"
+export MINT_MODE="full"
 export WGET="wget --quiet --no-check-certificate"
 
-go get github.com/go-ini/ini
-
 ./create-data-files.sh
 ./preinstall.sh
 

buildscripts/verify-build.sh

@@ -319,11 +319,10 @@ function __init__()
     mkdir -p "$MINIO_CONFIG_DIR"
     mkdir -p "$MINT_DATA_DIR"
 
-    if ! go get -u github.com/minio/mc; then
-        echo "failed to download https://github.com/minio/mc"
-        exit 1
-    fi
-    /bin/cp -a "$(go env GOPATH)"/bin/mc "$WORK_DIR/mc"
+   git clone https://github.com/minio/mc 
+   cd mc
+   go build 
+   /bin/cp -a ./mc "$WORK_DIR/mc"
 
     chmod a+x "$WORK_DIR/mc"
 

cmd/admin-handlers.go

@@ -177,11 +177,12 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 // ServerProperties holds some server information such as, version, region
 // uptime, etc..
 type ServerProperties struct {
-	Uptime   time.Duration `json:"uptime"`
-	Version  string        `json:"version"`
-	CommitID string        `json:"commitID"`
-	Region   string        `json:"region"`
-	SQSARN   []string      `json:"sqsARN"`
+	Uptime       time.Duration `json:"uptime"`
+	Version      string        `json:"version"`
+	CommitID     string        `json:"commitID"`
+	DeploymentID string        `json:"deploymentID"`
+	Region       string        `json:"region"`
+	SQSARN       []string      `json:"sqsARN"`
 }
 
 // ServerConnStats holds transferred bytes from/to the server
@@ -256,11 +257,12 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			ConnStats:   globalConnStats.toServerConnStats(),
 			HTTPStats:   globalHTTPStats.toServerHTTPStats(),
 			Properties: ServerProperties{
-				Uptime:   UTCNow().Sub(globalBootTime),
-				Version:  Version,
-				CommitID: CommitID,
-				SQSARN:   globalNotificationSys.GetARNList(),
-				Region:   globalServerConfig.GetRegion(),
+				Uptime:       UTCNow().Sub(globalBootTime),
+				Version:      Version,
+				CommitID:     CommitID,
+				DeploymentID: globalDeploymentID,
+				SQSARN:       globalNotificationSys.GetARNList(),
+				Region:       globalServerConfig.GetRegion(),
 			},
 		},
 	})
@@ -319,9 +321,8 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 	}
 
 	vars := mux.Vars(r)
-	perfType := vars["perfType"]
-
-	if perfType == "drive" {
+	switch perfType := vars["perfType"]; perfType {
+	case "drive":
 		info := objectAPI.StorageInfo(ctx)
 		if !(info.Backend.Type == BackendFS || info.Backend.Type == BackendErasure) {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
@@ -344,7 +345,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		// Reply with performance information (across nodes in a
 		// distributed setup) as json.
 		writeSuccessResponseJSON(w, jsonBytes)
-	} else if perfType == "cpu" {
+	case "cpu":
 		// Get CPU load details from local server's cpu(s)
 		cpu := localEndpointsCPULoad(globalEndpoints)
 		// Notify all other Minio peers to report cpu load numbers
@@ -361,7 +362,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		// Reply with cpu load information (across nodes in a
 		// distributed setup) as json.
 		writeSuccessResponseJSON(w, jsonBytes)
-	} else if perfType == "mem" {
+	case "mem":
 		// Get mem usage details from local server(s)
 		m := localEndpointsMemUsage(globalEndpoints)
 		// Notify all other Minio peers to report mem usage numbers
@@ -378,7 +379,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		// Reply with mem usage information (across nodes in a
 		// distributed setup) as json.
 		writeSuccessResponseJSON(w, jsonBytes)
-	} else {
+	default:
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 	}
 }

cmd/admin-heal-ops.go

@@ -21,14 +21,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"runtime"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
-	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 // healStatusSummary - overall short summary of a healing sequence
@@ -573,51 +571,22 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		marker := ""
-		isTruncated := true
-		for isTruncated {
-			if globalHTTPServer != nil {
-				// Wait at max 1 minute for an inprogress request
-				// before proceeding to heal
-				waitCount := 60
-				// Any requests in progress, delay the heal.
-				for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
-					waitCount--
-					time.Sleep(1 * time.Second)
-				}
+		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, func(bucket string, object string) error {
+			if h.isQuitting() {
+				return errHealStopSignalled
 			}
-
-			// Lists all objects under `config` prefix.
-			objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, metaPrefix,
-				marker, "", 1000)
-			if err != nil {
-				return errFnHealFromAPIErr(h.ctx, err)
+			res, herr := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove, h.settings.ScanMode)
+			// Object might have been deleted, by the time heal
+			// was attempted we ignore this object an move on.
+			if isErrObjectNotFound(herr) {
+				return nil
 			}
-
-			for index := range objectInfos.Objects {
-				if h.isQuitting() {
-					return errHealStopSignalled
-				}
-				o := objectInfos.Objects[index]
-				res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun, h.settings.Remove)
-				// Object might have been deleted, by the time heal
-				// was attempted we ignore this file an move on.
-				if isErrObjectNotFound(herr) {
-					continue
-				}
-				if herr != nil {
-					return herr
-				}
-				res.Type = madmin.HealItemBucketMetadata
-				if err = h.pushHealResultItem(res); err != nil {
-					return err
-				}
+			if herr != nil {
+				return herr
 			}
-
-			isTruncated = objectInfos.IsTruncated
-			marker = objectInfos.NextMarker
-		}
-		return nil
+			res.Type = madmin.HealItemBucketMetadata
+			return h.pushHealResultItem(res)
+		})
 	}
 }
 
@@ -720,46 +689,9 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
-	entries := runtime.NumCPU()
-
-	marker := ""
-	isTruncated := true
-	for isTruncated {
-		if globalHTTPServer != nil {
-			// Wait at max 1 minute for an inprogress request
-			// before proceeding to heal
-			waitCount := 60
-			// Any requests in progress, delay the heal.
-			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
-				waitCount--
-				time.Sleep(1 * time.Second)
-			}
-		}
-
-		// Heal numCPU * nodes objects at a time.
-		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
-			h.objPrefix, marker, "", entries)
-		if err != nil {
-			return errFnHealFromAPIErr(h.ctx, err)
-		}
-
-		g := errgroup.WithNErrs(len(objectInfos.Objects))
-		for index := range objectInfos.Objects {
-			index := index
-			g.Go(func() error {
-				o := objectInfos.Objects[index]
-				return h.healObject(o.Bucket, o.Name)
-			}, index)
-		}
-
-		for _, err := range g.Wait() {
-			if err != nil {
-				return err
-			}
-		}
-
-		isTruncated = objectInfos.IsTruncated
-		marker = objectInfos.NextMarker
+	if err = objectAPI.HealObjects(h.ctx, bucket,
+		h.objPrefix, h.healObject); err != nil {
+		return errFnHealFromAPIErr(h.ctx, err)
 	}
 	return nil
 }
@@ -770,13 +702,23 @@ func (h *healSequence) healObject(bucket, object string) error {
 		return errHealStopSignalled
 	}
 
+	if globalHTTPServer != nil {
+		// Wait at max 1 minute for an inprogress request
+		// before proceeding to heal
+		waitCount := 60
+		// Any requests in progress, delay the heal.
+		for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+			waitCount--
+			time.Sleep(1 * time.Second)
+		}
+	}
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
-	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove)
+	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove, h.settings.ScanMode)
 	if isErrObjectNotFound(err) {
 		return nil
 	}

cmd/api-errors.go

@@ -21,6 +21,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
@@ -114,7 +115,6 @@ const (
 	ErrInvalidRequestVersion
 	ErrMissingSignTag
 	ErrMissingSignHeadersTag
-	ErrPolicyAlreadyExpired
 	ErrMalformedDate
 	ErrMalformedPresignedDate
 	ErrMalformedCredentialDate
@@ -180,6 +180,7 @@ const (
 	// Minio extended errors.
 	ErrReadQuorum
 	ErrWriteQuorum
+	ErrParentIsObject
 	ErrStorageFull
 	ErrRequestBodyParse
 	ErrObjectExistsAsDirectory
@@ -625,11 +626,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Signature header missing SignedHeaders field.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrPolicyAlreadyExpired: {
-		Code:           "AccessDenied",
-		Description:    "Invalid according to Policy: Policy expired.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrMalformedExpires: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "X-Amz-Expires should be a number",
@@ -863,6 +859,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
 		HTTPStatusCode: http.StatusInsufficientStorage,
 	},
+	ErrParentIsObject: {
+		Code:           "XMinioParentIsObject",
+		Description:    "Object-prefix is already an object, please choose a different object-prefix name.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrRequestBodyParse: {
 		Code:           "XMinioRequestBodyParse",
 		Description:    "The request body failed to parse.",
@@ -1512,8 +1513,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrInvalidEncryptionMethod
 	case crypto.ErrInvalidCustomerAlgorithm:
 		apiErr = ErrInvalidSSECustomerAlgorithm
-	case crypto.ErrInvalidCustomerKey:
-		apiErr = ErrInvalidSSECustomerKey
 	case crypto.ErrMissingCustomerKey:
 		apiErr = ErrMissingSSECustomerKey
 	case crypto.ErrMissingCustomerKeyMD5:
@@ -1568,6 +1567,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectExistsAsDirectory
 	case PrefixAccessDenied:
 		apiErr = ErrAccessDenied
+	case ParentIsObject:
+		apiErr = ErrParentIsObject
 	case BucketNameInvalid:
 		apiErr = ErrInvalidBucketName
 	case BucketNotFound:
@@ -1647,11 +1648,25 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case crypto.Error:
 		apiErr = ErrObjectTampered
 	default:
-		apiErr = ErrInternalError
-		// Make sure to log the errors which we cannot translate
-		// to a meaningful S3 API errors. This is added to aid in
-		// debugging unexpected/unhandled errors.
-		logger.LogIf(ctx, err)
+		var ie, iw int
+		// This work-around is to handle the issue golang/go#30648
+		if _, ferr := fmt.Fscanf(strings.NewReader(err.Error()),
+			"request declared a Content-Length of %d but only wrote %d bytes",
+			&ie, &iw); ferr != nil {
+			apiErr = ErrInternalError
+			// Make sure to log the errors which we cannot translate
+			// to a meaningful S3 API errors. This is added to aid in
+			// debugging unexpected/unhandled errors.
+			logger.LogIf(ctx, err)
+		} else if ie > iw {
+			apiErr = ErrIncompleteBody
+		} else {
+			apiErr = ErrInternalError
+			// Make sure to log the errors which we cannot translate
+			// to a meaningful S3 API errors. This is added to aid in
+			// debugging unexpected/unhandled errors.
+			logger.LogIf(ctx, err)
+		}
 	}
 
 	return apiErr

cmd/api-errors_test.go

@@ -55,7 +55,7 @@ var toAPIErrorTests = []struct {
 	// SSE-C errors
 	{err: crypto.ErrInvalidCustomerAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
 	{err: crypto.ErrMissingCustomerKey, errCode: ErrMissingSSECustomerKey},
-	{err: crypto.ErrInvalidCustomerKey, errCode: ErrInvalidSSECustomerKey},
+	{err: crypto.ErrInvalidCustomerKey, errCode: ErrAccessDenied},
 	{err: crypto.ErrMissingCustomerKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
 	{err: crypto.ErrCustomerKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
 	{err: errObjectTampered, errCode: ErrObjectTampered},

cmd/api-headers.go

@@ -87,6 +87,9 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
 	}
 
+	if !objInfo.Expires.IsZero() {
+		w.Header().Set("Expires", objInfo.Expires.UTC().Format(http.TimeFormat))
+	}
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		if hasPrefix(k, ReservedMetadataPrefix) {

cmd/api-response.go

@@ -640,3 +640,38 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
+
+// writeCustomErrorResponseXML - similar to writeErrorResponse,
+// but accepts the error message directly (this allows messages to be
+// dynamically generated.)
+func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err APIError, errBody string, reqURL *url.URL, browser bool) {
+
+	switch err.Code {
+	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
+		// Set retry-after header to indicate user-agents to retry request after 120secs.
+		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+		w.Header().Set("Retry-After", "120")
+	case "AccessDenied":
+		// The request is from browser and also if browser
+		// is enabled we need to redirect.
+		if browser && globalIsBrowserEnabled {
+			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+			w.WriteHeader(http.StatusTemporaryRedirect)
+			return
+		}
+	}
+
+	reqInfo := logger.GetReqInfo(ctx)
+	errorResponse := APIErrorResponse{
+		Code:       err.Code,
+		Message:    errBody,
+		Resource:   reqURL.Path,
+		BucketName: reqInfo.BucketName,
+		Key:        reqInfo.ObjectName,
+		RequestID:  w.Header().Get(responseRequestIDKey),
+		HostID:     w.Header().Get(responseDeploymentIDKey),
+	}
+
+	encodedErrorResponse := encodeResponse(errorResponse)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
+}

cmd/auth-handler.go

@@ -32,7 +32,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -126,7 +126,7 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region stri
 		// We only support admin credentials to access admin APIs.
 
 		var owner bool
-		_, owner, s3Err = getReqAccessKeyV4(r, region)
+		_, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
 			return s3Err
 		}
@@ -136,7 +136,7 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region stri
 		}
 
 		// we only support V4 (no presign) with auth body
-		s3Err = isReqAuthenticated(ctx, r, region)
+		s3Err = isReqAuthenticated(ctx, r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
@@ -241,10 +241,10 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
 		}
-		if s3Err = isReqAuthenticated(ctx, r, region); s3Err != ErrNone {
+		if s3Err = isReqAuthenticated(ctx, r, region, serviceS3); s3Err != ErrNone {
 			return s3Err
 		}
-		cred, owner, s3Err = getReqAccessKeyV4(r, region)
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
 		return s3Err
@@ -314,21 +314,21 @@ func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) {
 	return doesPresignV2SignatureMatch(r)
 }
 
-func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode) {
-	sha256sum := getContentSha256Cksum(r)
+func reqSignatureV4Verify(r *http.Request, region string, stype serviceType) (s3Error APIErrorCode) {
+	sha256sum := getContentSha256Cksum(r, stype)
 	switch {
 	case isRequestSignatureV4(r):
-		return doesSignatureMatch(sha256sum, r, region)
+		return doesSignatureMatch(sha256sum, r, region, stype)
 	case isRequestPresignedSignatureV4(r):
-		return doesPresignedSignatureMatch(sha256sum, r, region)
+		return doesPresignedSignatureMatch(sha256sum, r, region, stype)
 	default:
 		return ErrAccessDenied
 	}
 }
 
 // Verify if request has valid AWS Signature Version '4'.
-func isReqAuthenticated(ctx context.Context, r *http.Request, region string) (s3Error APIErrorCode) {
-	if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
+func isReqAuthenticated(ctx context.Context, r *http.Request, region string, stype serviceType) (s3Error APIErrorCode) {
+	if errCode := reqSignatureV4Verify(r, region, stype); errCode != ErrNone {
 		return errCode
 	}
 
@@ -432,7 +432,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
 		region := globalServerConfig.GetRegion()
-		cred, owner, s3Err = getReqAccessKeyV4(r, region)
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
 		return s3Err

cmd/auth-handler_test.go

@@ -381,7 +381,8 @@ func TestIsReqAuthenticated(t *testing.T) {
 	ctx := context.Background()
 	// Validates all testcases.
 	for i, testCase := range testCases {
-		if s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
+		s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion(), serviceS3)
+		if s3Error != testCase.s3Error {
 			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
 				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
 			}

cmd/bucket-handlers.go

@@ -226,7 +226,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 				continue
 			}
 			bucketsInfo = append(bucketsInfo, BucketInfo{
-				Name:    strings.Trim(dnsRecord.Key, slashSeparator),
+				Name:    dnsRecord.Key,
 				Created: dnsRecord.CreationDate,
 			})
 			bucketSet.Add(dnsRecord.Key)
@@ -595,8 +595,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		}
 
 		// Make sure formValues adhere to policy restrictions.
-		if errCode = checkPostPolicy(formValues, postPolicyForm); errCode != ErrNone {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
+		if err = checkPostPolicy(formValues, postPolicyForm); err != nil {
+			writeCustomErrorResponseXML(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), err.Error(), r.URL, guessIsBrowserReq(r))
 			return
 		}
 

cmd/disk-cache-fs.go

@@ -332,7 +332,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	if isObjectDir(object, data.Size()) {
 		// Check if an object is present as one of the parent dir.
 		if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+			return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 		}
 		if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
@@ -350,7 +350,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-		return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
 	// Validate input data size and it can never be less than zero.

cmd/disk-cache.go

@@ -254,8 +254,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 
 	cleanupBackend := func() { bkReader.Close() }
 	cleanupPipe := func() { pipeReader.Close() }
-	gr = NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, cleanupBackend, cleanupPipe)
-	return gr, nil
+	return NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, opts.CheckCopyPrecondFn, cleanupBackend, cleanupPipe)
 }
 
 // Uses cached-object to serve the request. If object is not cached it serves the request from the backend and also
@@ -424,7 +423,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 	if delimiter == slashSeparator {
 		recursive = false
 	}
-	walkResultCh, endWalkCh := c.listPool.Release(listParams{bucket, recursive, marker, prefix, false})
+	walkResultCh, endWalkCh := c.listPool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
 		isLeaf := func(bucket, object string) bool {
@@ -496,7 +495,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 		}
 	}
 
-	params := listParams{bucket, recursive, nextMarker, prefix, false}
+	params := listParams{bucket, recursive, nextMarker, prefix}
 	if !eof {
 		c.listPool.Set(params, walkResultCh, endWalkCh)
 	}

cmd/fs-v1-metadata.go

@@ -22,8 +22,10 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	pathutil "path"
+	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
@@ -169,7 +171,15 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	} else {
 		objInfo.StorageClass = globalMinioDefaultStorageClass
 	}
-
+	var (
+		t time.Time
+		e error
+	)
+	if exp, ok := m.Meta["expires"]; ok {
+		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+			objInfo.Expires = t.UTC()
+		}
+	}
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.

cmd/fs-v1-metadata_test.go

@@ -38,6 +38,9 @@ func TestFSV1MetadataObjInfo(t *testing.T) {
 	if objInfo.IsDir {
 		t.Fatal("Unexpected object info value for IsDir", objInfo.IsDir)
 	}
+	if !objInfo.Expires.IsZero() {
+		t.Fatal("Unexpected object info value for Expires ", objInfo.Expires)
+	}
 }
 
 // TestReadFSMetadata - readFSMetadata testing with a healthy and faulty disk

cmd/fs-v1-multipart.go

@@ -487,7 +487,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(ctx, bucket, pathutil.Dir(object)) {
-		return oi, toObjectErr(errFileAccessDenied, bucket, object)
+		return oi, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
 	if _, err := fs.statBucketDir(ctx, bucket); err != nil {

cmd/fs-v1.go

@@ -30,6 +30,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
@@ -289,10 +290,8 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 	}
 	defer bucketLock.Unlock()
 	// Verify if bucket is valid.
-	if !IsValidBucketName(bucket) {
-		err := BucketNameInvalid{Bucket: bucket}
-		logger.LogIf(ctx, err)
-		return err
+	if s3utils.CheckValidBucketNameStrict(bucket) != nil {
+		return BucketNameInvalid{Bucket: bucket}
 	}
 	bucketDir, err := fs.getBucketDir(ctx, bucket)
 	if err != nil {
@@ -341,7 +340,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 
 	for _, entry := range entries {
 		// Ignore all reserved bucket names and invalid bucket names.
-		if isReservedOrInvalidBucket(entry) {
+		if isReservedOrInvalidBucket(entry, false) {
 			continue
 		}
 		var fi os.FileInfo
@@ -490,20 +489,18 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		}
 	}
 
-	// For a directory, we need to send an reader that returns no bytes.
-	if hasSuffix(object, slashSeparator) {
-		// The lock taken above is released when
-		// objReader.Close() is called by the caller.
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), ObjectInfo{}, nsUnlocker), nil
-	}
-
 	// Otherwise we get the object info
 	var objInfo ObjectInfo
 	if objInfo, err = fs.getObjectInfo(ctx, bucket, object); err != nil {
 		nsUnlocker()
 		return nil, toObjectErr(err, bucket, object)
 	}
-
+	// For a directory, we need to send an reader that returns no bytes.
+	if hasSuffix(object, slashSeparator) {
+		// The lock taken above is released when
+		// objReader.Close() is called by the caller.
+		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+	}
 	// Take a rwPool lock for NFS gateway type deployment
 	rwPoolUnlocker := func() {}
 	if bucket != minioMetaBucket && lockType != noLock {
@@ -519,7 +516,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		rwPoolUnlocker = func() { fs.rwPool.Close(fsMetaPath) }
 	}
 
-	objReaderFn, off, length, rErr := NewGetObjectReader(rs, objInfo, nsUnlocker, rwPoolUnlocker)
+	objReaderFn, off, length, rErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn, nsUnlocker, rwPoolUnlocker)
 	if rErr != nil {
 		return nil, rErr
 	}
@@ -547,7 +544,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		return nil, err
 	}
 
-	return objReaderFn(reader, h, closeFn)
+	return objReaderFn(reader, h, opts.CheckCopyPrecondFn, closeFn)
 }
 
 // GetObject - reads an object from the disk.
@@ -842,8 +839,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	if isObjectDir(object, data.Size()) {
 		// Check if an object is present as one of the parent dir.
 		if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-			logger.LogIf(ctx, errFileAccessDenied)
-			return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+			return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 		}
 		if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
 			logger.LogIf(ctx, err)
@@ -862,8 +858,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-		logger.LogIf(ctx, errFileAccessDenied)
-		return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
 	// Validate input data size and it can never be less than zero.
@@ -1151,8 +1146,7 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 		return fs.getObjectInfo(ctx, bucket, entry)
 	}
 
-	heal := false // true only for xl.ListObjectsHeal()
-	walkResultCh, endWalkCh := fs.listPool.Release(listParams{bucket, recursive, marker, prefix, heal})
+	walkResultCh, endWalkCh := fs.listPool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
 		isLeaf := func(bucket, object string) bool {
@@ -1206,7 +1200,7 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 	}
 
 	// Save list routine for the next marker if we haven't reached EOF.
-	params := listParams{bucket, recursive, nextMarker, prefix, heal}
+	params := listParams{bucket, recursive, nextMarker, prefix}
 	if !eof {
 		fs.listPool.Set(params, walkResultCh, endWalkCh)
 	}
@@ -1244,7 +1238,7 @@ func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRe
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (
+func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (
 	res madmin.HealResultItem, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return res, NotImplemented{}
@@ -1257,10 +1251,10 @@ func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remo
 	return madmin.HealResultItem{}, NotImplemented{}
 }
 
-// ListObjectsHeal - list all objects to be healed. Valid only for XL
-func (fs *FSObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
+// HealObjects - no-op for fs. Valid only for XL.
+func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, fn func(string, string) error) (e error) {
 	logger.LogIf(ctx, NotImplemented{})
-	return loi, NotImplemented{}
+	return NotImplemented{}
 }
 
 // ListBucketsHeal - list all buckets to be healed. Valid only for XL

cmd/fs-v1_test.go

@@ -22,6 +22,8 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // Tests for if parent directory is object
@@ -228,8 +230,8 @@ func TestFSPutObject(t *testing.T) {
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backend corruption occurred")
 	}
-	if nerr, ok := err.(PrefixAccessDenied); !ok {
-		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
+	if nerr, ok := err.(ParentIsObject); !ok {
+		t.Fatalf("Expected ParentIsObject, got %#v", err)
 	} else {
 		if nerr.Bucket != "bucket" {
 			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
@@ -243,8 +245,8 @@ func TestFSPutObject(t *testing.T) {
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backned corruption occurred")
 	}
-	if nerr, ok := err.(PrefixAccessDenied); !ok {
-		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
+	if nerr, ok := err.(ParentIsObject); !ok {
+		t.Fatalf("Expected ParentIsObject, got %#v", err)
 	} else {
 		if nerr.Bucket != "bucket" {
 			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
@@ -390,19 +392,19 @@ func TestFSHealObject(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.HealObject(context.Background(), "bucket", "object", false, false)
+	_, err := obj.HealObject(context.Background(), "bucket", "object", false, false, madmin.HealDeepScan)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}
 }
 
-// TestFSListObjectHeal - tests for fs ListObjectHeals
-func TestFSListObjectsHeal(t *testing.T) {
+// TestFSHealObjects - tests for fs HealObjects to return not implemented.
+func TestFSHealObjects(t *testing.T) {
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.ListObjectsHeal(context.Background(), "bucket", "prefix", "marker", "delimiter", 1000)
+	err := obj.HealObjects(context.Background(), "bucket", "prefix", nil)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-common.go

@@ -166,6 +166,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 		ContentType:     oi.ContentType,
 		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
 		StorageClass:    oi.StorageClass,
+		Expires:         oi.Expires,
 	}
 }
 

cmd/gateway-unsupported.go

@@ -102,7 +102,7 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 }
 
 // HealObject - Not implemented stub
-func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (h madmin.HealResultItem, e error) {
+func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}
 }
 
@@ -111,9 +111,9 @@ func (a GatewayUnsupported) ListObjectsV2(ctx context.Context, bucket, prefix, c
 	return result, NotImplemented{}
 }
 
-// ListObjectsHeal - Not implemented stub
-func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	return loi, NotImplemented{}
+// HealObjects - Not implemented stub
+func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, fn func(string, string) error) (e error) {
+	return NotImplemented{}
 }
 
 // CopyObject copies a blob from source container to destination container.

cmd/gateway/azure/gateway-azure.go

@@ -651,7 +651,7 @@ func (a *azureObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject - reads an object from azure. Supports additional
@@ -829,6 +829,9 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 // CopyObject - Copies a blob from source container to destination container.
 // Uses Azure equivalent CopyBlob API.
 func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.ObjectInfo{}, minio.PreConditionFailed{}
+	}
 	srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL()
 	destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject)
 	azureMeta, props, err := s3MetaToAzureProperties(ctx, srcInfo.UserDefined)

cmd/gateway/b2/gateway-b2.go

@@ -417,7 +417,7 @@ func (l *b2Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject reads an object from B2. Supports additional

cmd/gateway/gcs/gateway-gcs.go

@@ -753,7 +753,7 @@ func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string,
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject - reads an object from GCS. Supports additional
@@ -797,12 +797,22 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
 	// All google cloud storage objects have a CRC32c hash, whereas composite objects may not have a MD5 hash
 	// Refer https://cloud.google.com/storage/docs/hashes-etags. Use CRC32C for ETag
 	metadata := make(map[string]string)
+	var (
+		expiry time.Time
+		e      error
+	)
 	for k, v := range attrs.Metadata {
 		k = http.CanonicalHeaderKey(k)
 		// Translate the GCS custom metadata prefix
 		if strings.HasPrefix(k, "X-Goog-Meta-") {
 			k = strings.Replace(k, "X-Goog-Meta-", "X-Amz-Meta-", 1)
 		}
+		if k == "Expires" {
+			if expiry, e = time.Parse(http.TimeFormat, v); e == nil {
+				expiry = expiry.UTC()
+			}
+			continue
+		}
 		metadata[k] = v
 	}
 	if attrs.ContentType != "" {
@@ -829,6 +839,7 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
 		UserDefined:     metadata,
 		ContentType:     attrs.ContentType,
 		ContentEncoding: attrs.ContentEncoding,
+		Expires:         expiry,
 	}
 }
 
@@ -918,7 +929,9 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
 // CopyObject - Copies a blob from source container to destination container.
 func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
 	srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (minio.ObjectInfo, error) {
-
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.ObjectInfo{}, minio.PreConditionFailed{}
+	}
 	src := l.client.Bucket(srcBucket).Object(srcObject)
 	dst := l.client.Bucket(destBucket).Object(destObject)
 

cmd/gateway/oss/gateway-oss.go

@@ -26,10 +26,11 @@ import (
 	"strings"
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-	"github.com/dustin/go-humanize"
+	humanize "github.com/dustin/go-humanize"
 
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio-go/pkg/s3utils"
 	minio "github.com/minio/minio/cmd"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -181,7 +182,7 @@ func appendS3MetaToOSSOptions(ctx context.Context, opts []oss.Option, s3Metadata
 		case k == "X-Amz-Acl":
 			// Valid values: public-read, private, and public-read-write
 			opts = append(opts, oss.ObjectACL(oss.ACLType(v)))
-		case k == "X-Amz-Server-Side​-Encryption":
+		case k == "X-Amz-Server-Side-Encryption":
 			opts = append(opts, oss.ServerSideEncryption(v))
 		case k == "X-Amz-Copy-Source-If-Match":
 			opts = append(opts, oss.CopySourceIfMatch(v))
@@ -349,7 +350,7 @@ func ossIsValidBucketName(bucket string) bool {
 	if strings.Contains(bucket, ".") {
 		return false
 	}
-	if !minio.IsValidBucketName(bucket) {
+	if s3utils.CheckValidBucketNameStrict(bucket) != nil {
 		return false
 	}
 	return true
@@ -569,7 +570,7 @@ func (l *ossObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject reads an object on OSS. Supports additional
@@ -664,6 +665,9 @@ func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, r *mi
 
 // CopyObject copies an object from source bucket to a destination bucket.
 func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.ObjectInfo{}, minio.PreConditionFailed{}
+	}
 	bkt, err := l.Client.Bucket(srcBucket)
 	if err != nil {
 		logger.LogIf(ctx, err)

cmd/gateway/s3/gateway-s3-metadata.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"net/http"
 	"time"
 
 	minio "github.com/minio/minio/cmd"
@@ -83,6 +84,7 @@ func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
 		"Content-Length",
 		"Last-Modified",
 		"Content-Type",
+		"Expires",
 	}, defaultFilterKeys...)
 	objInfo := minio.ObjectInfo{
 		IsDir:           false,
@@ -100,6 +102,15 @@ func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
 	if sc, ok := m.Meta["x-amz-storage-class"]; ok {
 		objInfo.StorageClass = sc
 	}
+	var (
+		t time.Time
+		e error
+	)
+	if exp, ok := m.Meta["expires"]; ok {
+		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+			objInfo.Expires = t.UTC()
+		}
+	}
 	// Success.
 	return objInfo
 }

cmd/gateway/s3/gateway-s3-sse.go

@@ -313,7 +313,7 @@ func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		return l.s3Objects.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	objInfo.UserDefined = minio.CleanMinioInternalMetadataKeys(objInfo.UserDefined)
-	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo)
+	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo, o.CheckCopyPrecondFn)
 	if err != nil {
 		return nil, minio.ErrorRespToObjectError(err)
 	}
@@ -329,7 +329,7 @@ func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return fn(pr, h, pipeCloser)
+	return fn(pr, h, o.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo

cmd/gateway/s3/gateway-s3.go

@@ -400,7 +400,7 @@ func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject reads an object from S3. Supports additional
@@ -463,6 +463,9 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string,
 
 // CopyObject copies an object from source bucket to a destination bucket.
 func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.ObjectInfo{}, minio.PreConditionFailed{}
+	}
 	// Set this header such that following CopyObject() always sets the right metadata on the destination.
 	// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
 	// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
@@ -533,6 +536,9 @@ func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object str
 // existing object or a part of it.
 func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
 	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.PartInfo{}, minio.PreConditionFailed{}
+	}
 	srcInfo.UserDefined = map[string]string{
 		"x-amz-copy-source-if-match": srcInfo.ETag,
 	}

cmd/generic-handlers.go

@@ -383,7 +383,23 @@ type resourceHandler struct {
 
 // setCorsHandler handler for CORS (Cross Origin Resource Sharing)
 func setCorsHandler(h http.Handler) http.Handler {
-	c := cors.AllowAll()
+
+	c := cors.New(cors.Options{
+		AllowedOrigins: []string{"*"},
+		AllowedMethods: []string{
+			http.MethodGet,
+			http.MethodPut,
+			http.MethodHead,
+			http.MethodPost,
+			http.MethodDelete,
+			http.MethodOptions,
+			http.MethodPatch,
+		},
+		AllowedHeaders:   []string{"*"},
+		ExposedHeaders:   []string{"*"},
+		AllowCredentials: true,
+	})
+
 	return c.Handler(h)
 }
 

cmd/globals.go

@@ -57,7 +57,6 @@ const (
 	globalMinioDefaultStorageClass = "STANDARD"
 	globalWindowsOSName            = "windows"
 	globalNetBSDOSName             = "netbsd"
-	globalSolarisOSName            = "solaris"
 	globalMinioModeFS              = "mode-server-fs"
 	globalMinioModeXL              = "mode-server-xl"
 	globalMinioModeDistXL          = "mode-server-distributed-xl"

cmd/handler-utils.go

@@ -184,7 +184,7 @@ func getRedirectPostRawQuery(objInfo ObjectInfo) string {
 
 // Returns access credentials in the request Authorization header.
 func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
-	cred, _, _ = getReqAccessKeyV4(r, region)
+	cred, _, _ = getReqAccessKeyV4(r, region, serviceS3)
 	if cred.AccessKey == "" {
 		cred, _, _ = getReqAccessKeyV2(r)
 	}

cmd/http/server.go

@@ -25,6 +25,8 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	"golang.org/x/net/http2"
+
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/pkg/certs"
 )
@@ -46,6 +48,9 @@ const (
 
 	// DefaultMaxHeaderBytes - default maximum HTTP header size in bytes.
 	DefaultMaxHeaderBytes = 1 * humanize.MiByte
+
+	// DefaultHTTP2MaxConcurrentStreams - default value for HTTP 2.0 maximum concurrent streams allowed.
+	DefaultHTTP2MaxConcurrentStreams = 1024
 )
 
 // Server - extended http.Server supports multiple addresses to serve and enhanced connection handling.
@@ -56,7 +61,7 @@ type Server struct {
 	TCPKeepAliveTimeout    time.Duration // timeout used for underneath TCP connection.
 	UpdateBytesReadFunc    func(int)     // function to be called to update bytes read in bufConn.
 	UpdateBytesWrittenFunc func(int)     // function to be called to update bytes written in bufConn.
-	listenerMutex          *sync.Mutex   // to guard 'listener' field.
+	listenerMutex          sync.Mutex    // to guard 'listener' field.
 	listener               *httpListener // HTTP listener for all 'Addrs' field.
 	inShutdown             uint32        // indicates whether the server is in shutdown or not
 	requestCount           int32         // counter holds no. of request in progress.
@@ -120,6 +125,11 @@ func (srv *Server) Start() (err error) {
 
 	// Start servicing with listener.
 	if tlsConfig != nil {
+		if err = http2.ConfigureServer(&srv.Server, &http2.Server{
+			MaxConcurrentStreams: DefaultHTTP2MaxConcurrentStreams,
+		}); err != nil {
+			return err
+		}
 		return srv.Server.Serve(tls.NewListener(listener, tlsConfig))
 	}
 	return srv.Server.Serve(listener)
@@ -202,7 +212,6 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 		Addrs:               addrs,
 		ShutdownTimeout:     DefaultShutdownTimeout,
 		TCPKeepAliveTimeout: DefaultTCPKeepAliveTimeout,
-		listenerMutex:       &sync.Mutex{},
 	}
 	httpServer.Handler = handler
 	httpServer.TLSConfig = tlsConfig

cmd/http/server_test.go

@@ -77,10 +77,6 @@ func TestNewServer(t *testing.T) {
 			t.Fatalf("Case %v: server.TCPKeepAliveTimeout: expected: %v, got: %v", (i + 1), DefaultTCPKeepAliveTimeout, server.TCPKeepAliveTimeout)
 		}
 
-		if server.listenerMutex == nil {
-			t.Fatalf("Case %v: server.listenerMutex: expected: <non-nil>, got: <nil>", (i + 1))
-		}
-
 		if server.ReadTimeout != DefaultReadTimeout {
 			t.Fatalf("Case %v: server.ReadTimeout: expected: %v, got: %v", (i + 1), DefaultReadTimeout, server.ReadTimeout)
 		}

cmd/iam.go

@@ -28,7 +28,7 @@ import (
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -326,6 +326,27 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 	return nil
 }
 
+// GetUserPolicy - returns canned policy name associated with a user.
+func (sys *IAMSys) GetUserPolicy(accessKey string) (policyName string, err error) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return "", errServerNotInitialized
+	}
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	if _, ok := sys.iamUsersMap[accessKey]; !ok {
+		return "", errNoSuchUser
+	}
+
+	if _, ok := sys.iamPolicyMap[accessKey]; !ok {
+		return "", errNoSuchUser
+	}
+
+	return sys.iamPolicyMap[accessKey], nil
+}
+
 // ListUsers - list all users.
 func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 	objectAPI := newObjectLayerFn()

cmd/notification.go

@@ -43,7 +43,7 @@ type NotificationSys struct {
 	targetList                 *event.TargetList
 	bucketRulesMap             map[string]event.RulesMap
 	bucketRemoteTargetRulesMap map[string]map[event.TargetID]event.RulesMap
-	peerRPCClientMap           map[xnet.Host]*PeerRPCClient
+	peerClients                []*peerRESTClient
 }
 
 // GetARNList - returns available ARNs.
@@ -63,11 +63,6 @@ func (sys *NotificationSys) GetARNList() []string {
 	return arns
 }
 
-// GetPeerRPCClient - returns PeerRPCClient of addr.
-func (sys *NotificationSys) GetPeerRPCClient(addr xnet.Host) *PeerRPCClient {
-	return sys.peerRPCClientMap[addr]
-}
-
 // NotificationPeerErr returns error associated for a remote peer.
 type NotificationPeerErr struct {
 	Host xnet.Host // Remote host on which the rpc call was initiated
@@ -78,15 +73,15 @@ type NotificationPeerErr struct {
 func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string) {
 	go func() {
 		var wg sync.WaitGroup
-		for addr, client := range sys.peerRPCClientMap {
+		for _, client := range sys.peerClients {
 			wg.Add(1)
-			go func(addr xnet.Host, client *PeerRPCClient) {
+			go func(client *peerRESTClient) {
 				defer wg.Done()
 				if err := client.DeleteBucket(bucketName); err != nil {
-					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name)
 					logger.LogIf(ctx, err)
 				}
-			}(addr, client)
+			}(client)
 		}
 		wg.Wait()
 	}()
@@ -146,52 +141,58 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
 	}()
 }
 
-// ReloadFormat - calls ReloadFormat RPC call on all peers.
+// ReloadFormat - calls ReloadFormat REST call on all peers.
 func (sys *NotificationSys) ReloadFormat(dryRun bool) []NotificationPeerErr {
-	var idx = 0
-	ng := WithNPeers(len(sys.peerRPCClientMap))
-	for addr, client := range sys.peerRPCClientMap {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
 		client := client
 		ng.Go(context.Background(), func() error {
 			return client.ReloadFormat(dryRun)
-		}, idx, addr)
-		idx++
+		}, idx, *client.host)
 	}
 	return ng.Wait()
 }
 
 // LoadUsers - calls LoadUsers RPC call on all peers.
 func (sys *NotificationSys) LoadUsers() []NotificationPeerErr {
-	var idx = 0
-	ng := WithNPeers(len(sys.peerRPCClientMap))
-	for addr, client := range sys.peerRPCClientMap {
-		ng.Go(context.Background(), client.LoadUsers, idx, addr)
-		idx++
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(context.Background(), client.LoadUsers, idx, *client.host)
 	}
 	return ng.Wait()
 }
 
 // LoadCredentials - calls LoadCredentials RPC call on all peers.
 func (sys *NotificationSys) LoadCredentials() []NotificationPeerErr {
-	var idx = 0
-	ng := WithNPeers(len(sys.peerRPCClientMap))
-	for addr, client := range sys.peerRPCClientMap {
-		ng.Go(context.Background(), client.LoadCredentials, idx, addr)
-		idx++
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(context.Background(), client.LoadCredentials, idx, *client.host)
 	}
 	return ng.Wait()
 }
 
 // StartProfiling - start profiling on remote peers, by initiating a remote RPC.
 func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr {
-	var idx = 0
-	ng := WithNPeers(len(sys.peerRPCClientMap))
-	for addr, client := range sys.peerRPCClientMap {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
 		client := client
 		ng.Go(context.Background(), func() error {
 			return client.StartProfiling(profiler)
-		}, idx, addr)
-		idx++
+		}, idx, *client.host)
 	}
 	return ng.Wait()
 }
@@ -205,10 +206,13 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 	zipWriter := zip.NewWriter(writer)
 	defer zipWriter.Close()
 
-	for addr, client := range sys.peerRPCClientMap {
-		data, err := client.DownloadProfilingData()
+	for _, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		data, err := client.DownloadProfileData()
 		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 			ctx := logger.SetReqInfo(ctx, reqInfo)
 			logger.LogIf(ctx, err)
 			continue
@@ -218,7 +222,7 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 
 		// Send profiling data to zip as file
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
-			name:    fmt.Sprintf("profiling-%s.pprof", addr),
+			name:    fmt.Sprintf("profiling-%s.pprof", client.host.String()),
 			size:    int64(len(data)),
 			mode:    0600,
 			modTime: UTCNow(),
@@ -226,20 +230,20 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 			sys:     nil,
 		})
 		if zerr != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 			ctx := logger.SetReqInfo(ctx, reqInfo)
 			logger.LogIf(ctx, zerr)
 			continue
 		}
 		zwriter, zerr := zipWriter.CreateHeader(header)
 		if zerr != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 			ctx := logger.SetReqInfo(ctx, reqInfo)
 			logger.LogIf(ctx, zerr)
 			continue
 		}
 		if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 			ctx := logger.SetReqInfo(ctx, reqInfo)
 			logger.LogIf(ctx, err)
 			continue
@@ -289,45 +293,48 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 
 // SignalService - calls signal service RPC call on all peers.
 func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerErr {
-	var idx = 0
-	ng := WithNPeers(len(sys.peerRPCClientMap))
-	for addr, client := range sys.peerRPCClientMap {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
 		client := client
 		ng.Go(context.Background(), func() error {
 			return client.SignalService(sig)
-		}, idx, addr)
-		idx++
+		}, idx, *client.host)
 	}
 	return ng.Wait()
 }
 
 // ServerInfo - calls ServerInfo RPC call on all peers.
 func (sys *NotificationSys) ServerInfo(ctx context.Context) []ServerInfo {
-	var idx = 0
-	serverInfo := make([]ServerInfo, len(sys.peerRPCClientMap))
+	serverInfo := make([]ServerInfo, len(sys.peerClients))
 	var wg sync.WaitGroup
-	for addr, client := range sys.peerRPCClientMap {
+	for index, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
 		wg.Add(1)
-		go func(idx int, addr xnet.Host, client *PeerRPCClient) {
+		go func(idx int, client *peerRESTClient) {
 			defer wg.Done()
 			// Try to fetch serverInfo remotely in three attempts.
 			for i := 0; i < 3; i++ {
 				info, err := client.ServerInfo()
 				if err == nil {
 					serverInfo[idx] = ServerInfo{
-						Addr: addr.String(),
+						Addr: client.host.String(),
 						Data: &info,
 					}
 					return
 				}
 				serverInfo[idx] = ServerInfo{
-					Addr:  addr.String(),
+					Addr:  client.host.String(),
 					Data:  &info,
 					Error: err.Error(),
 				}
 				// Last iteration log the error.
 				if i == 2 {
-					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 					ctx := logger.SetReqInfo(ctx, reqInfo)
 					logger.LogIf(ctx, err)
 				}
@@ -336,8 +343,7 @@ func (sys *NotificationSys) ServerInfo(ctx context.Context) []ServerInfo {
 					time.Sleep(1 * time.Second)
 				}
 			}
-		}(idx, addr, client)
-		idx++
+		}(index, client)
 	}
 	wg.Wait()
 	return serverInfo
@@ -345,19 +351,22 @@ func (sys *NotificationSys) ServerInfo(ctx context.Context) []ServerInfo {
 
 // GetLocks - makes GetLocks RPC call on all peers.
 func (sys *NotificationSys) GetLocks(ctx context.Context) []*PeerLocks {
-	var idx = 0
-	locksResp := make([]*PeerLocks, len(sys.peerRPCClientMap))
+
+	locksResp := make([]*PeerLocks, len(sys.peerClients))
 	var wg sync.WaitGroup
-	for addr, client := range sys.peerRPCClientMap {
+	for index, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
 		wg.Add(1)
-		go func(idx int, addr xnet.Host, client *PeerRPCClient) {
+		go func(idx int, client *peerRESTClient) {
 			defer wg.Done()
 			// Try to fetch serverInfo remotely in three attempts.
 			for i := 0; i < 3; i++ {
 				serverLocksResp, err := client.GetLocks()
 				if err == nil {
 					locksResp[idx] = &PeerLocks{
-						Addr:  addr.String(),
+						Addr:  client.host.String(),
 						Locks: serverLocksResp,
 					}
 					return
@@ -365,17 +374,16 @@ func (sys *NotificationSys) GetLocks(ctx context.Context) []*PeerLocks {
 
 				// Last iteration log the error.
 				if i == 2 {
-					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 					ctx := logger.SetReqInfo(ctx, reqInfo)
-					logger.LogOnceIf(ctx, err, addr.String())
+					logger.LogOnceIf(ctx, err, client.host.String())
 				}
 				// Wait for one second and no need wait after last attempt.
 				if i < 2 {
 					time.Sleep(1 * time.Second)
 				}
 			}
-		}(idx, addr, client)
-		idx++
+		}(index, client)
 	}
 	wg.Wait()
 	return locksResp
@@ -385,15 +393,18 @@ func (sys *NotificationSys) GetLocks(ctx context.Context) []*PeerLocks {
 func (sys *NotificationSys) SetBucketPolicy(ctx context.Context, bucketName string, bucketPolicy *policy.Policy) {
 	go func() {
 		var wg sync.WaitGroup
-		for addr, client := range sys.peerRPCClientMap {
+		for _, client := range sys.peerClients {
+			if client == nil {
+				continue
+			}
 			wg.Add(1)
-			go func(addr xnet.Host, client *PeerRPCClient) {
+			go func(client *peerRESTClient) {
 				defer wg.Done()
 				if err := client.SetBucketPolicy(bucketName, bucketPolicy); err != nil {
-					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name)
 					logger.LogIf(ctx, err)
 				}
-			}(addr, client)
+			}(client)
 		}
 		wg.Wait()
 	}()
@@ -403,15 +414,18 @@ func (sys *NotificationSys) SetBucketPolicy(ctx context.Context, bucketName stri
 func (sys *NotificationSys) RemoveBucketPolicy(ctx context.Context, bucketName string) {
 	go func() {
 		var wg sync.WaitGroup
-		for addr, client := range sys.peerRPCClientMap {
+		for _, client := range sys.peerClients {
+			if client == nil {
+				continue
+			}
 			wg.Add(1)
-			go func(addr xnet.Host, client *PeerRPCClient) {
+			go func(client *peerRESTClient) {
 				defer wg.Done()
 				if err := client.RemoveBucketPolicy(bucketName); err != nil {
-					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name)
 					logger.LogIf(ctx, err)
 				}
-			}(addr, client)
+			}(client)
 		}
 		wg.Wait()
 	}()
@@ -421,15 +435,18 @@ func (sys *NotificationSys) RemoveBucketPolicy(ctx context.Context, bucketName s
 func (sys *NotificationSys) PutBucketNotification(ctx context.Context, bucketName string, rulesMap event.RulesMap) {
 	go func() {
 		var wg sync.WaitGroup
-		for addr, client := range sys.peerRPCClientMap {
+		for _, client := range sys.peerClients {
+			if client == nil {
+				continue
+			}
 			wg.Add(1)
-			go func(addr xnet.Host, client *PeerRPCClient, rulesMap event.RulesMap) {
+			go func(client *peerRESTClient, rulesMap event.RulesMap) {
 				defer wg.Done()
 				if err := client.PutBucketNotification(bucketName, rulesMap); err != nil {
-					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name)
 					logger.LogIf(ctx, err)
 				}
-			}(addr, client, rulesMap.Clone())
+			}(client, rulesMap.Clone())
 		}
 		wg.Wait()
 	}()
@@ -440,15 +457,18 @@ func (sys *NotificationSys) ListenBucketNotification(ctx context.Context, bucket
 	targetID event.TargetID, localPeer xnet.Host) {
 	go func() {
 		var wg sync.WaitGroup
-		for addr, client := range sys.peerRPCClientMap {
+		for _, client := range sys.peerClients {
+			if client == nil {
+				continue
+			}
 			wg.Add(1)
-			go func(addr xnet.Host, client *PeerRPCClient) {
+			go func(client *peerRESTClient) {
 				defer wg.Done()
 				if err := client.ListenBucketNotification(bucketName, eventNames, pattern, targetID, localPeer); err != nil {
-					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name)
 					logger.LogIf(ctx, err)
 				}
-			}(addr, client)
+			}(client)
 		}
 		wg.Wait()
 	}()
@@ -492,7 +512,17 @@ func (sys *NotificationSys) RemoteTargetExist(bucketName string, targetID event.
 	return ok
 }
 
-// initListeners - initializes PeerRPC clients available in listener.json.
+// ListenBucketNotificationArgs - listen bucket notification RPC arguments.
+type ListenBucketNotificationArgs struct {
+	AuthArgs   `json:"-"`
+	BucketName string         `json:"-"`
+	EventNames []event.Name   `json:"eventNames"`
+	Pattern    string         `json:"pattern"`
+	TargetID   event.TargetID `json:"targetId"`
+	Addr       xnet.Host      `json:"addr"`
+}
+
+// initListeners - initializes PeerREST clients available in listener.json.
 func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLayer, bucketName string) error {
 	// listener.json is available/applicable only in DistXL mode.
 	if !globalIsDistXL {
@@ -542,12 +572,12 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 			continue
 		}
 
-		rpcClient := sys.GetPeerRPCClient(args.Addr)
-		if rpcClient == nil {
-			return fmt.Errorf("unable to find PeerRPCClient by address %v in listener.json for bucket %v", args.Addr, bucketName)
+		client, err := newPeerRESTClient(&args.Addr)
+		if err != nil {
+			return fmt.Errorf("unable to find PeerHost by address %v in listener.json for bucket %v", args.Addr, bucketName)
 		}
 
-		exist, err := rpcClient.RemoteTargetExist(bucketName, args.TargetID)
+		exist, err := client.RemoteTargetExist(bucketName, args.TargetID)
 		if err != nil {
 			logger.GetReqInfo(ctx).AppendTags("targetID", args.TargetID.Name)
 			logger.LogIf(ctx, err)
@@ -558,7 +588,7 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 			continue
 		}
 
-		target := NewPeerRPCClientTarget(bucketName, args.TargetID, rpcClient)
+		target := NewPeerRESTClientTarget(bucketName, args.TargetID, client)
 		rulesMap := event.NewRulesMap(args.EventNames, args.Pattern, target.ID())
 		if err = sys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
 			logger.GetReqInfo(ctx).AppendTags("targetName", target.id.Name)
@@ -721,24 +751,25 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
 
 // DrivePerfInfo - Drive speed (read and write) information
 func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo {
-	reply := make([]ServerDrivesPerfInfo, len(sys.peerRPCClientMap))
+	reply := make([]ServerDrivesPerfInfo, len(sys.peerClients))
 	var wg sync.WaitGroup
-	var i int
-	for addr, client := range sys.peerRPCClientMap {
+	for i, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
 		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient, idx int) {
+		go func(client *peerRESTClient, idx int) {
 			defer wg.Done()
 			di, err := client.DrivePerfInfo()
 			if err != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr.String())
+				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", client.host.String())
 				ctx := logger.SetReqInfo(context.Background(), reqInfo)
 				logger.LogIf(ctx, err)
-				di.Addr = addr.String()
+				di.Addr = client.host.String()
 				di.Error = err.Error()
 			}
 			reply[idx] = di
-		}(addr, client, i)
-		i++
+		}(client, i)
 	}
 	wg.Wait()
 	return reply
@@ -746,24 +777,25 @@ func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo {
 
 // MemUsageInfo - Mem utilization information
 func (sys *NotificationSys) MemUsageInfo() []ServerMemUsageInfo {
-	reply := make([]ServerMemUsageInfo, len(sys.peerRPCClientMap))
+	reply := make([]ServerMemUsageInfo, len(sys.peerClients))
 	var wg sync.WaitGroup
-	var i int
-	for addr, client := range sys.peerRPCClientMap {
+	for i, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
 		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient, idx int) {
+		go func(client *peerRESTClient, idx int) {
 			defer wg.Done()
 			memi, err := client.MemUsageInfo()
 			if err != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr.String())
+				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", client.host.String())
 				ctx := logger.SetReqInfo(context.Background(), reqInfo)
 				logger.LogIf(ctx, err)
-				memi.Addr = addr.String()
+				memi.Addr = client.host.String()
 				memi.Error = err.Error()
 			}
 			reply[idx] = memi
-		}(addr, client, i)
-		i++
+		}(client, i)
 	}
 	wg.Wait()
 	return reply
@@ -771,24 +803,25 @@ func (sys *NotificationSys) MemUsageInfo() []ServerMemUsageInfo {
 
 // CPULoadInfo - CPU utilization information
 func (sys *NotificationSys) CPULoadInfo() []ServerCPULoadInfo {
-	reply := make([]ServerCPULoadInfo, len(sys.peerRPCClientMap))
+	reply := make([]ServerCPULoadInfo, len(sys.peerClients))
 	var wg sync.WaitGroup
-	var i int
-	for addr, client := range sys.peerRPCClientMap {
+	for i, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
 		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient, idx int) {
+		go func(client *peerRESTClient, idx int) {
 			defer wg.Done()
 			cpui, err := client.CPULoadInfo()
 			if err != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr.String())
+				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", client.host.String())
 				ctx := logger.SetReqInfo(context.Background(), reqInfo)
 				logger.LogIf(ctx, err)
-				cpui.Addr = addr.String()
+				cpui.Addr = client.host.String()
 				cpui.Error = err.Error()
 			}
 			reply[idx] = cpui
-		}(addr, client, i)
-		i++
+		}(client, i)
 	}
 	wg.Wait()
 	return reply
@@ -797,14 +830,18 @@ func (sys *NotificationSys) CPULoadInfo() []ServerCPULoadInfo {
 // NewNotificationSys - creates new notification system object.
 func NewNotificationSys(config *serverConfig, endpoints EndpointList) *NotificationSys {
 	targetList := getNotificationTargets(config)
-	peerRPCClientMap := makeRemoteRPCClients(endpoints)
+	remoteHosts := getRemoteHosts(endpoints)
+	remoteClients, err := getRestClients(remoteHosts)
+	if err != nil {
+		logger.FatalIf(err, "Unable to start notification sub system")
+	}
 
 	// bucketRulesMap/bucketRemoteTargetRulesMap are initialized by NotificationSys.Init()
 	return &NotificationSys{
 		targetList:                 targetList,
 		bucketRulesMap:             make(map[string]event.RulesMap),
 		bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
-		peerRPCClientMap:           peerRPCClientMap,
+		peerClients:                remoteClients,
 	}
 }
 

cmd/object-api-datatypes.go

@@ -96,6 +96,9 @@ type ObjectInfo struct {
 	// by the Content-Type header field.
 	ContentEncoding string
 
+	// Date and time at which the object is no longer able to be cached
+	Expires time.Time
+
 	// Specify object storage class
 	StorageClass string
 

cmd/object-api-errors.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"fmt"
 	"io"
+	"path"
 )
 
 // Converts underlying storage error. Convenience function written to
@@ -47,6 +48,13 @@ func toObjectErr(err error, params ...string) error {
 				Object: params[1],
 			}
 		}
+	case errFileParentIsFile:
+		if len(params) >= 2 {
+			err = ParentIsObject{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		}
 	case errIsNotRegular:
 		if len(params) >= 2 {
 			err = ObjectExistsAsDirectory{
@@ -182,6 +190,13 @@ func (e PrefixAccessDenied) Error() string {
 	return "Prefix access is denied: " + e.Bucket + "/" + e.Object
 }
 
+// ParentIsObject object access is denied.
+type ParentIsObject GenericError
+
+func (e ParentIsObject) Error() string {
+	return "Parent is object " + e.Bucket + "/" + path.Dir(e.Object)
+}
+
 // BucketExists bucket exists.
 type BucketExists GenericError
 
@@ -383,3 +398,15 @@ func isErrObjectNotFound(err error) bool {
 	_, ok := err.(ObjectNotFound)
 	return ok
 }
+
+// PreConditionFailed - Check if copy precondition failed
+type PreConditionFailed struct{}
+
+func (e PreConditionFailed) Error() string {
+	return "At least one of the pre-conditions you specified did not hold"
+}
+
+func isErrPreconditionFailed(err error) bool {
+	_, ok := err.(PreConditionFailed)
+	return ok
+}

cmd/object-api-interface.go

@@ -26,10 +26,14 @@ import (
 	"github.com/minio/minio/pkg/policy"
 )
 
+// CheckCopyPreconditionFn returns true if copy precondition check failed.
+type CheckCopyPreconditionFn func(o ObjectInfo, encETag string) bool
+
 // ObjectOptions represents object options for ObjectLayer operations
 type ObjectOptions struct {
 	ServerSideEncryption encrypt.ServerSide
 	UserDefined          map[string]string
+	CheckCopyPrecondFn   CheckCopyPreconditionFn
 }
 
 // LockType represents required locking for ObjectLayer operations
@@ -84,9 +88,9 @@ type ObjectLayer interface {
 	ReloadFormat(ctx context.Context, dryRun bool) error
 	HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error)
 	HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem, error)
-	HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (madmin.HealResultItem, error)
+	HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (madmin.HealResultItem, error)
 	ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error)
-	ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error)
+	HealObjects(ctx context.Context, bucket, prefix string, healObjectFn func(string, string) error) error
 
 	// Policy operations
 	SetBucketPolicy(context.Context, string, *policy.Policy) error

cmd/object-api-utils.go

@@ -34,6 +34,7 @@ import (
 	"unicode/utf8"
 
 	snappy "github.com/golang/snappy"
+	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
@@ -207,8 +208,8 @@ func getCompleteMultipartMD5(ctx context.Context, parts []CompletePart) (string,
 func cleanMetadata(metadata map[string]string) map[string]string {
 	// Remove STANDARD StorageClass
 	metadata = removeStandardStorageClass(metadata)
-	// Clean meta etag keys 'md5Sum', 'etag'.
-	return cleanMetadataKeys(metadata, "md5Sum", "etag")
+	// Clean meta etag keys 'md5Sum', 'etag', "expires".
+	return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires")
 }
 
 // Filter X-Amz-Storage-Class field only if it is set to STANDARD.
@@ -273,10 +274,16 @@ func isStringEqual(s1 string, s2 string) bool {
 }
 
 // Ignores all reserved bucket names or invalid bucket names.
-func isReservedOrInvalidBucket(bucketEntry string) bool {
+func isReservedOrInvalidBucket(bucketEntry string, strict bool) bool {
 	bucketEntry = strings.TrimSuffix(bucketEntry, slashSeparator)
-	if !IsValidBucketName(bucketEntry) {
-		return true
+	if strict {
+		if err := s3utils.CheckValidBucketNameStrict(bucketEntry); err != nil {
+			return true
+		}
+	} else {
+		if err := s3utils.CheckValidBucketName(bucketEntry); err != nil {
+			return true
+		}
 	}
 	return isMinioMetaBucket(bucketEntry) || isMinioReservedBucket(bucketEntry)
 }
@@ -422,30 +429,41 @@ type GetObjectReader struct {
 	pReader io.Reader
 
 	cleanUpFns []func()
+	precondFn  func(ObjectInfo, string) bool
 	once       sync.Once
 }
 
 // NewGetObjectReaderFromReader sets up a GetObjectReader with a given
 // reader. This ignores any object properties.
-func NewGetObjectReaderFromReader(r io.Reader, oi ObjectInfo, cleanupFns ...func()) *GetObjectReader {
+func NewGetObjectReaderFromReader(r io.Reader, oi ObjectInfo, pcfn CheckCopyPreconditionFn, cleanupFns ...func()) (*GetObjectReader, error) {
+	if pcfn != nil {
+		if ok := pcfn(oi, ""); ok {
+			// Call the cleanup funcs
+			for i := len(cleanupFns) - 1; i >= 0; i-- {
+				cleanupFns[i]()
+			}
+			return nil, PreConditionFailed{}
+		}
+	}
 	return &GetObjectReader{
 		ObjInfo:    oi,
 		pReader:    r,
 		cleanUpFns: cleanupFns,
-	}
+		precondFn:  pcfn,
+	}, nil
 }
 
 // ObjReaderFn is a function type that takes a reader and returns
 // GetObjectReader and an error. Request headers are passed to provide
 // encryption parameters. cleanupFns allow cleanup funcs to be
 // registered for calling after usage of the reader.
-type ObjReaderFn func(inputReader io.Reader, h http.Header, cleanupFns ...func()) (r *GetObjectReader, err error)
+type ObjReaderFn func(inputReader io.Reader, h http.Header, pcfn CheckCopyPreconditionFn, cleanupFns ...func()) (r *GetObjectReader, err error)
 
 // NewGetObjectReader creates a new GetObjectReader. The cleanUpFns
 // are called on Close() in reverse order as passed here. NOTE: It is
 // assumed that clean up functions do not panic (otherwise, they may
 // not all run!).
-func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func()) (
+func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, pcfn CheckCopyPreconditionFn, cleanUpFns ...func()) (
 	fn ObjReaderFn, off, length int64, err error) {
 
 	// Call the clean-up functions immediately in case of exit
@@ -486,7 +504,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func())
 		// a reader that returns the desired range of
 		// encrypted bytes. The header parameter is used to
 		// provide encryption parameters.
-		fn = func(inputReader io.Reader, h http.Header, cFns ...func()) (r *GetObjectReader, err error) {
+		fn = func(inputReader io.Reader, h http.Header, pcfn CheckCopyPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
 			copySource := h.Get(crypto.SSECopyAlgorithm) != ""
 
 			cFns = append(cleanUpFns, cFns...)
@@ -501,9 +519,18 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func())
 				}
 				return nil, err
 			}
+			encETag := oi.ETag
+			oi.ETag = getDecryptedETag(h, oi, copySource) // Decrypt the ETag before top layer consumes this value.
 
-			// Decrypt the ETag before top layer consumes this value.
-			oi.ETag = getDecryptedETag(h, oi, copySource)
+			if pcfn != nil {
+				if ok := pcfn(oi, encETag); ok {
+					// Call the cleanup funcs
+					for i := len(cFns) - 1; i >= 0; i-- {
+						cFns[i]()
+					}
+					return nil, PreConditionFailed{}
+				}
+			}
 
 			// Apply the skipLen and limit on the
 			// decrypted stream
@@ -514,6 +541,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func())
 				ObjInfo:    oi,
 				pReader:    decReader,
 				cleanUpFns: cFns,
+				precondFn:  pcfn,
 			}
 			return r, nil
 		}
@@ -545,7 +573,17 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func())
 				return nil, 0, 0, errInvalidRange
 			}
 		}
-		fn = func(inputReader io.Reader, _ http.Header, cFns ...func()) (r *GetObjectReader, err error) {
+		fn = func(inputReader io.Reader, _ http.Header, pcfn CheckCopyPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
+			cFns = append(cleanUpFns, cFns...)
+			if pcfn != nil {
+				if ok := pcfn(oi, ""); ok {
+					// Call the cleanup funcs
+					for i := len(cFns) - 1; i >= 0; i-- {
+						cFns[i]()
+					}
+					return nil, PreConditionFailed{}
+				}
+			}
 			// Decompression reader.
 			snappyReader := snappy.NewReader(inputReader)
 			// Apply the skipLen and limit on the
@@ -557,7 +595,8 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func())
 			r = &GetObjectReader{
 				ObjInfo:    oi,
 				pReader:    decReader,
-				cleanUpFns: append(cleanUpFns, cFns...),
+				cleanUpFns: cFns,
+				precondFn:  pcfn,
 			}
 			return r, nil
 		}
@@ -567,11 +606,22 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func())
 		if err != nil {
 			return nil, 0, 0, err
 		}
-		fn = func(inputReader io.Reader, _ http.Header, cFns ...func()) (r *GetObjectReader, err error) {
+		fn = func(inputReader io.Reader, _ http.Header, pcfn CheckCopyPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
+			cFns = append(cleanUpFns, cFns...)
+			if pcfn != nil {
+				if ok := pcfn(oi, ""); ok {
+					// Call the cleanup funcs
+					for i := len(cFns) - 1; i >= 0; i-- {
+						cFns[i]()
+					}
+					return nil, PreConditionFailed{}
+				}
+			}
 			r = &GetObjectReader{
 				ObjInfo:    oi,
 				pReader:    inputReader,
-				cleanUpFns: append(cleanUpFns, cFns...),
+				cleanUpFns: cFns,
+				precondFn:  pcfn,
 			}
 			return r, nil
 		}
@@ -675,3 +725,54 @@ func CleanMinioInternalMetadataKeys(metadata map[string]string) map[string]strin
 	}
 	return newMeta
 }
+
+// snappyCompressReader compresses data as it reads
+// from the underlying io.Reader.
+type snappyCompressReader struct {
+	r      io.Reader
+	w      *snappy.Writer
+	closed bool
+	buf    bytes.Buffer
+}
+
+func newSnappyCompressReader(r io.Reader) *snappyCompressReader {
+	cr := &snappyCompressReader{r: r}
+	cr.w = snappy.NewBufferedWriter(&cr.buf)
+	return cr
+}
+
+func (cr *snappyCompressReader) Read(p []byte) (int, error) {
+	if cr.closed {
+		// if snappy writer is closed r has been completely read,
+		// return any remaining data in buf.
+		return cr.buf.Read(p)
+	}
+
+	// read from original using p as buffer
+	nr, readErr := cr.r.Read(p)
+
+	// write read bytes to snappy writer
+	nw, err := cr.w.Write(p[:nr])
+	if err != nil {
+		return 0, err
+	}
+	if nw != nr {
+		return 0, io.ErrShortWrite
+	}
+
+	// if last of data from reader, close snappy writer to flush
+	if readErr == io.EOF {
+		err := cr.w.Close()
+		cr.closed = true
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	// read compressed bytes out of buf
+	n, err := cr.buf.Read(p)
+	if readErr != io.EOF && (err == nil || err == io.EOF) {
+		err = readErr
+	}
+	return n, err
+}

cmd/object-api-utils_test.go

@@ -17,10 +17,14 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"io"
 	"net/http"
 	"reflect"
 	"testing"
+
+	"github.com/golang/snappy"
 )
 
 // Tests validate bucket name.
@@ -544,3 +548,58 @@ func TestGetCompressedOffsets(t *testing.T) {
 		}
 	}
 }
+
+func TestSnappyCompressReader(t *testing.T) {
+	tests := []struct {
+		name string
+		data []byte
+	}{
+		{name: "empty", data: nil},
+		{name: "small", data: []byte("hello, world")},
+		{name: "large", data: bytes.Repeat([]byte("hello, world"), 1000)},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case
+
+			r := newSnappyCompressReader(bytes.NewReader(tt.data))
+
+			var rdrBuf bytes.Buffer
+			_, err := io.CopyBuffer(&rdrBuf, r, buf)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var stdBuf bytes.Buffer
+			w := snappy.NewBufferedWriter(&stdBuf)
+			_, err = io.CopyBuffer(w, bytes.NewReader(tt.data), buf)
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = w.Close()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var (
+				got  = rdrBuf.Bytes()
+				want = stdBuf.Bytes()
+			)
+			if !bytes.Equal(got, want) {
+				t.Errorf("encoded data does not match\n\t%q\n\t%q", got, want)
+			}
+
+			var decBuf bytes.Buffer
+			decRdr := snappy.NewReader(&rdrBuf)
+			_, err = io.Copy(&decBuf, decRdr)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !bytes.Equal(tt.data, decBuf.Bytes()) {
+				t.Errorf("roundtrip failed\n\t%q\n\t%q", tt.data, decBuf.Bytes())
+			}
+		})
+	}
+}

cmd/object-handlers-common.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 )
@@ -33,8 +34,8 @@ import (
 //  x-amz-copy-source-if-unmodified-since
 //  x-amz-copy-source-if-match
 //  x-amz-copy-source-if-none-match
-func checkCopyObjectPartPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
-	return checkCopyObjectPreconditions(ctx, w, r, objInfo)
+func checkCopyObjectPartPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo, encETag string) bool {
+	return checkCopyObjectPreconditions(ctx, w, r, objInfo, encETag)
 }
 
 // Validates the preconditions for CopyObject, returns true if CopyObject operation should not proceed.
@@ -43,11 +44,14 @@ func checkCopyObjectPartPreconditions(ctx context.Context, w http.ResponseWriter
 //  x-amz-copy-source-if-unmodified-since
 //  x-amz-copy-source-if-match
 //  x-amz-copy-source-if-none-match
-func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
+func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo, encETag string) bool {
 	// Return false for methods other than GET and HEAD.
 	if r.Method != "PUT" {
 		return false
 	}
+	if encETag == "" {
+		encETag = objInfo.ETag
+	}
 	// If the object doesn't have a modtime (IsZero), or the modtime
 	// is obviously garbage (Unix time == 0), then ignore modtimes
 	// and don't process the If-Modified-Since header.
@@ -95,11 +99,16 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
 		}
 	}
 
+	ssec := crypto.SSECopy.IsRequested(r.Header)
 	// x-amz-copy-source-if-match : Return the object only if its entity tag (ETag) is the
 	// same as the one specified; otherwise return a 412 (precondition failed).
 	ifMatchETagHeader := r.Header.Get("x-amz-copy-source-if-match")
 	if ifMatchETagHeader != "" {
-		if objInfo.ETag != "" && !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
+		etag := objInfo.ETag
+		if ssec {
+			etag = encETag[len(encETag)-32:]
+		}
+		if objInfo.ETag != "" && !isETagEqual(etag, ifMatchETagHeader) {
 			// If the object ETag does not match with the specified ETag.
 			writeHeaders()
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL, guessIsBrowserReq(r))
@@ -111,7 +120,11 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
 	// one specified otherwise, return a 304 (not modified).
 	ifNoneMatchETagHeader := r.Header.Get("x-amz-copy-source-if-none-match")
 	if ifNoneMatchETagHeader != "" {
-		if objInfo.ETag != "" && isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) {
+		etag := objInfo.ETag
+		if ssec {
+			etag = encETag[len(encETag)-32:]
+		}
+		if objInfo.ETag != "" && isETagEqual(etag, ifNoneMatchETagHeader) {
 			// If the object ETag matches with the specified ETag.
 			writeHeaders()
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL, guessIsBrowserReq(r))

cmd/object-handlers.go

@@ -33,7 +33,6 @@ import (
 
 	"time"
 
-	snappy "github.com/golang/snappy"
 	"github.com/gorilla/mux"
 	miniogo "github.com/minio/minio-go"
 	"github.com/minio/minio-go/pkg/encrypt"
@@ -761,21 +760,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	if !cpSrcDstSame {
 		lock = readLock
 	}
-
+	checkCopyPrecondFn := func(o ObjectInfo, encETag string) bool {
+		return checkCopyObjectPreconditions(ctx, w, r, o, encETag)
+	}
+	getOpts.CheckCopyPrecondFn = checkCopyPrecondFn
+	srcOpts.CheckCopyPrecondFn = checkCopyPrecondFn
 	var rs *HTTPRangeSpec
 	gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, lock, getOpts)
 	if err != nil {
+		if isErrPreconditionFailed(err) {
+			return
+		}
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	defer gr.Close()
 	srcInfo := gr.ObjInfo
 
-	// Verify before x-amz-copy-source preconditions before continuing with CopyObject.
-	if checkCopyObjectPreconditions(ctx, w, r, srcInfo) {
-		return
-	}
-
 	/// maximum Upload size for object in a single CopyObject operation.
 	if isMaxObjectSize(srcInfo.Size) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL, guessIsBrowserReq(r))
@@ -831,21 +832,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		// Remove all source encrypted related metadata to
 		// avoid copying them in target object.
 		crypto.RemoveInternalEntries(srcInfo.UserDefined)
-		// Open a pipe for compression.
-		// Where pipeWriter is piped to srcInfo.Reader.
-		// gr writes to pipeWriter.
-		pipeReader, pipeWriter := io.Pipe()
-		reader = pipeReader
+
+		reader = newSnappyCompressReader(gr)
 		length = -1
-
-		snappyWriter := snappy.NewBufferedWriter(pipeWriter)
-
-		go func() {
-			// Compress the decompressed source object.
-			_, cerr := io.Copy(snappyWriter, gr)
-			snappyWriter.Close()
-			pipeWriter.CloseWithError(cerr)
-		}()
 	} else {
 		// Remove the metadata for remote calls.
 		delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression")
@@ -1200,12 +1189,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 
 	case authTypePresigned, authTypeSigned:
-		if s3Err = reqSignatureV4Verify(r, globalServerConfig.GetRegion()); s3Err != ErrNone {
+		if s3Err = reqSignatureV4Verify(r, globalServerConfig.GetRegion(), serviceS3); s3Err != ErrNone {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 		if !skipContentSha256Cksum(r) {
-			sha256hex = getContentSha256Cksum(r)
+			sha256hex = getContentSha256Cksum(r, serviceS3)
 		}
 	}
 
@@ -1216,28 +1205,17 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
 		metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
 
-		pipeReader, pipeWriter := io.Pipe()
-		snappyWriter := snappy.NewBufferedWriter(pipeWriter)
-
-		var actualReader *hash.Reader
-		actualReader, err = hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
-		go func() {
-			// Writing to the compressed writer.
-			_, cerr := io.CopyN(snappyWriter, actualReader, actualSize)
-			snappyWriter.Close()
-			pipeWriter.CloseWithError(cerr)
-		}()
-
 		// Set compression metrics.
+		reader = newSnappyCompressReader(actualReader)
 		size = -1   // Since compressed size is un-predictable.
 		md5hex = "" // Do not try to verify the content.
 		sha256hex = ""
-		reader = pipeReader
 	}
 
 	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
@@ -1597,9 +1575,17 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 		}
 	}
+	checkCopyPartPrecondFn := func(o ObjectInfo, encETag string) bool {
+		return checkCopyObjectPartPreconditions(ctx, w, r, o, encETag)
+	}
+	getOpts.CheckCopyPrecondFn = checkCopyPartPrecondFn
+	srcOpts.CheckCopyPrecondFn = checkCopyPartPrecondFn
 
 	gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, readLock, getOpts)
 	if err != nil {
+		if isErrPreconditionFailed(err) {
+			return
+		}
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1621,11 +1607,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Verify before x-amz-copy-source preconditions before continuing with CopyObject.
-	if checkCopyObjectPartPreconditions(ctx, w, r, srcInfo) {
-		return
-	}
-
 	// Get the object offset & length
 	startOffset, length, err := rs.GetOffsetLength(actualPartSize)
 	if err != nil {
@@ -1654,21 +1635,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	isCompressed := compressPart
 	// Compress only if the compression is enabled during initial multipart.
 	if isCompressed {
-		// Open a pipe for compression.
-		// Where pipeWriter is piped to srcInfo.Reader.
-		// gr writes to pipeWriter.
-		pipeReader, pipeWriter := io.Pipe()
-		reader = pipeReader
+		reader = newSnappyCompressReader(gr)
 		length = -1
-
-		snappyWriter := snappy.NewBufferedWriter(pipeWriter)
-
-		go func() {
-			// Compress the decompressed source object.
-			_, cerr := io.Copy(snappyWriter, gr)
-			snappyWriter.Close()
-			pipeWriter.CloseWithError(cerr)
-		}()
 	} else {
 		reader = gr
 	}
@@ -1868,19 +1836,17 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			return
 		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error = reqSignatureV4Verify(r, globalServerConfig.GetRegion()); s3Error != ErrNone {
+		if s3Error = reqSignatureV4Verify(r, globalServerConfig.GetRegion(), serviceS3); s3Error != ErrNone {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		if !skipContentSha256Cksum(r) {
-			sha256hex = getContentSha256Cksum(r)
+			sha256hex = getContentSha256Cksum(r, serviceS3)
 		}
 	}
 
 	actualSize := size
-	var pipeReader *io.PipeReader
-	var pipeWriter *io.PipeWriter
 
 	// get encryption options
 	var opts ObjectOptions
@@ -1902,28 +1868,17 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 	isCompressed := false
 	if objectAPI.IsCompressionSupported() && compressPart {
-		pipeReader, pipeWriter = io.Pipe()
-		snappyWriter := snappy.NewBufferedWriter(pipeWriter)
-
-		var actualReader *hash.Reader
-		actualReader, err = hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
-		go func() {
-			// Writing to the compressed writer.
-			_, cerr := io.CopyN(snappyWriter, actualReader, actualSize)
-			snappyWriter.Close()
-			pipeWriter.CloseWithError(cerr)
-		}()
-
 		// Set compression metrics.
+		reader = newSnappyCompressReader(actualReader)
 		size = -1   // Since compressed size is un-predictable.
 		md5hex = "" // Do not try to verify the content.
 		sha256hex = ""
-		reader = pipeReader
 		isCompressed = true
 	}
 
@@ -2016,7 +1971,6 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 	etag := partInfo.ETag
 	if isCompressed {
-		pipeWriter.Close()
 		// Suppress compressed ETag.
 		etag = partInfo.ETag + "-1"
 	} else if isEncrypted {

cmd/os-reliable.go

@@ -142,6 +142,10 @@ func renameAll(srcFilePath, dstFilePath string) (err error) {
 			return fmt.Errorf("%s (%s)->(%s)", errCrossDeviceLink, srcFilePath, dstFilePath)
 		case os.IsNotExist(err):
 			return errFileNotFound
+		case os.IsExist(err):
+			// This is returned only when destination is a directory and we
+			// are attempting a rename from file to directory.
+			return errIsNotRegular
 		default:
 			return err
 		}

cmd/peer-rest-client-target.go

@@ -18,35 +18,35 @@ package cmd
 
 import "github.com/minio/minio/pkg/event"
 
-// PeerRPCClientTarget - RPCClient is an event.Target which sends event to target of remote peer.
-type PeerRPCClientTarget struct {
+// PeerRESTClientTarget - RPCClient is an event.Target which sends event to target of remote peer.
+type PeerRESTClientTarget struct {
 	id             event.TargetID
 	remoteTargetID event.TargetID
-	rpcClient      *PeerRPCClient
+	restClient     *peerRESTClient
 	bucketName     string
 }
 
 // ID - returns target ID.
-func (target *PeerRPCClientTarget) ID() event.TargetID {
+func (target *PeerRESTClientTarget) ID() event.TargetID {
 	return target.id
 }
 
 // Send - sends event to remote peer by making RPC call.
-func (target *PeerRPCClientTarget) Send(eventData event.Event) error {
-	return target.rpcClient.SendEvent(target.bucketName, target.id, target.remoteTargetID, eventData)
+func (target *PeerRESTClientTarget) Send(eventData event.Event) error {
+	return target.restClient.SendEvent(target.bucketName, target.id, target.remoteTargetID, eventData)
 }
 
 // Close - does nothing and available for interface compatibility.
-func (target *PeerRPCClientTarget) Close() error {
+func (target *PeerRESTClientTarget) Close() error {
 	return nil
 }
 
-// NewPeerRPCClientTarget - creates RPCClient target with given target ID available in remote peer.
-func NewPeerRPCClientTarget(bucketName string, targetID event.TargetID, rpcClient *PeerRPCClient) *PeerRPCClientTarget {
-	return &PeerRPCClientTarget{
+// NewPeerRESTClientTarget - creates RPCClient target with given target ID available in remote peer.
+func NewPeerRESTClientTarget(bucketName string, targetID event.TargetID, restClient *peerRESTClient) *PeerRESTClientTarget {
+	return &PeerRESTClientTarget{
 		id:             event.TargetID{ID: targetID.ID, Name: targetID.Name + "+" + mustGetUUID()},
 		remoteTargetID: targetID,
 		bucketName:     bucketName,
-		rpcClient:      rpcClient,
+		restClient:     restClient,
 	}
 }

cmd/peer-rest-client.go

@@ -0,0 +1,425 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/gob"
+	"io"
+	"net/url"
+
+	"github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/rest"
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/policy"
+)
+
+// client to talk to peer Nodes.
+type peerRESTClient struct {
+	host       *xnet.Host
+	restClient *rest.Client
+	connected  bool
+}
+
+// Reconnect to a peer rest server.
+func (client *peerRESTClient) reConnect() error {
+	// correct (intelligent) retry logic will be
+	// implemented in subsequent PRs.
+	client.connected = true
+	return nil
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
+// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *peerRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+	if !client.connected {
+		err := client.reConnect()
+		logger.LogIf(context.Background(), err)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if values == nil {
+		values = make(url.Values)
+	}
+
+	respBody, err = client.restClient.Call(method, values, body, length)
+	if err == nil {
+		return respBody, nil
+	}
+
+	if isNetworkError(err) {
+		client.connected = false
+	}
+
+	return nil, err
+}
+
+// Stringer provides a canonicalized representation of node.
+func (client *peerRESTClient) String() string {
+	return client.host.String()
+}
+
+// IsOnline - returns whether RPC client failed to connect or not.
+func (client *peerRESTClient) IsOnline() bool {
+	return client.connected
+}
+
+// Close - marks the client as closed.
+func (client *peerRESTClient) Close() error {
+	client.connected = false
+	client.restClient.Close()
+	return nil
+}
+
+// GetLocksResp stores various info from the client for each lock that is requested.
+type GetLocksResp map[string][]lockRequesterInfo
+
+// GetLocks - fetch older locks for a remote node.
+func (client *peerRESTClient) GetLocks() (locks GetLocksResp, err error) {
+	respBody, err := client.call(peerRESTMethodGetLocks, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&locks)
+	return locks, err
+}
+
+// ServerInfo - fetch server information for a remote node.
+func (client *peerRESTClient) ServerInfo() (info ServerInfoData, err error) {
+	respBody, err := client.call(peerRESTMethodServerInfo, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
+// CPULoadInfo - fetch CPU information for a remote node.
+func (client *peerRESTClient) CPULoadInfo() (info ServerCPULoadInfo, err error) {
+	respBody, err := client.call(peerRESTMethodCPULoadInfo, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
+// DrivePerfInfo - fetch Drive performance information for a remote node.
+func (client *peerRESTClient) DrivePerfInfo() (info ServerDrivesPerfInfo, err error) {
+	respBody, err := client.call(peerRESTMethodDrivePerfInfo, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
+// MemUsageInfo - fetch memory usage information for a remote node.
+func (client *peerRESTClient) MemUsageInfo() (info ServerMemUsageInfo, err error) {
+	respBody, err := client.call(peerRESTMethodMemUsageInfo, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
+// StartProfiling - Issues profiling command on the peer node.
+func (client *peerRESTClient) StartProfiling(profiler string) error {
+	values := make(url.Values)
+	values.Set(peerRESTProfiler, profiler)
+	respBody, err := client.call(peerRESTMethodStartProfiling, values, nil, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// DownloadProfileData - download profiled data from a remote node.
+func (client *peerRESTClient) DownloadProfileData() (data []byte, err error) {
+	respBody, err := client.call(peerRESTMethodDownloadProfilingData, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&data)
+	return data, err
+}
+
+// DeleteBucket - Delete notification and policies related to the bucket.
+func (client *peerRESTClient) DeleteBucket(bucket string) error {
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+	respBody, err := client.call(peerRESTMethodDeleteBucket, values, nil, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// ReloadFormat - reload format on the peer node.
+func (client *peerRESTClient) ReloadFormat(dryRun bool) error {
+	values := make(url.Values)
+	if dryRun {
+		values.Set(peerRESTDryRun, "true")
+	} else {
+		values.Set(peerRESTDryRun, "false")
+	}
+
+	respBody, err := client.call(peerRESTMethodReloadFormat, values, nil, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// ListenBucketNotification - send listent bucket notification to peer nodes.
+func (client *peerRESTClient) ListenBucketNotification(bucket string, eventNames []event.Name,
+	pattern string, targetID event.TargetID, addr xnet.Host) error {
+	args := listenBucketNotificationReq{
+		EventNames: eventNames,
+		Pattern:    pattern,
+		TargetID:   targetID,
+		Addr:       addr,
+	}
+
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+
+	var reader bytes.Buffer
+	err := gob.NewEncoder(&reader).Encode(args)
+	if err != nil {
+		return err
+	}
+
+	respBody, err := client.call(peerRESTMethodBucketNotificationListen, values, &reader, -1)
+	if err != nil {
+		return err
+	}
+
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// SendEvent - calls send event RPC.
+func (client *peerRESTClient) SendEvent(bucket string, targetID, remoteTargetID event.TargetID, eventData event.Event) error {
+	args := sendEventRequest{
+		TargetID: remoteTargetID,
+		Event:    eventData,
+	}
+
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+
+	var reader bytes.Buffer
+	err := gob.NewEncoder(&reader).Encode(args)
+	if err != nil {
+		return err
+	}
+	respBody, err := client.call(peerRESTMethodSendEvent, values, &reader, -1)
+	if err != nil {
+		return err
+	}
+
+	var eventResp sendEventResp
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&eventResp)
+
+	if err != nil || !eventResp.Success {
+		reqInfo := &logger.ReqInfo{BucketName: bucket}
+		reqInfo.AppendTags("targetID", targetID.Name)
+		reqInfo.AppendTags("event", eventData.EventName.String())
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+		globalNotificationSys.RemoveRemoteTarget(bucket, targetID)
+	}
+
+	return err
+}
+
+// RemoteTargetExist - calls remote target ID exist REST API.
+func (client *peerRESTClient) RemoteTargetExist(bucket string, targetID event.TargetID) (bool, error) {
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+
+	var reader bytes.Buffer
+	err := gob.NewEncoder(&reader).Encode(targetID)
+	if err != nil {
+		return false, err
+	}
+
+	respBody, err := client.call(peerRESTMethodTargetExists, values, &reader, -1)
+	if err != nil {
+		return false, err
+	}
+	defer http.DrainBody(respBody)
+	var targetExists remoteTargetExistsResp
+	err = gob.NewDecoder(respBody).Decode(&targetExists)
+	return targetExists.Exists, err
+}
+
+// RemoveBucketPolicy - Remove bucket policy on the peer node.
+func (client *peerRESTClient) RemoveBucketPolicy(bucket string) error {
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+	respBody, err := client.call(peerRESTMethodBucketPolicyRemove, values, nil, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// SetBucketPolicy - Set bucket policy on the peer node.
+func (client *peerRESTClient) SetBucketPolicy(bucket string, bucketPolicy *policy.Policy) error {
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+
+	var reader bytes.Buffer
+	err := gob.NewEncoder(&reader).Encode(bucketPolicy)
+	if err != nil {
+		return err
+	}
+
+	respBody, err := client.call(peerRESTMethodBucketPolicySet, values, &reader, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// PutBucketNotification - Put bucket notification on the peer node.
+func (client *peerRESTClient) PutBucketNotification(bucket string, rulesMap event.RulesMap) error {
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+
+	var reader bytes.Buffer
+	err := gob.NewEncoder(&reader).Encode(&rulesMap)
+	if err != nil {
+		return err
+	}
+
+	respBody, err := client.call(peerRESTMethodBucketNotificationPut, values, &reader, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// LoadUsers - send load users command to peer nodes.
+func (client *peerRESTClient) LoadUsers() (err error) {
+	respBody, err := client.call(peerRESTMethodLoadUsers, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// LoadCredentials - send load credentials command to peer nodes.
+func (client *peerRESTClient) LoadCredentials() (err error) {
+	respBody, err := client.call(peerRESTMethodLoadCredentials, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// SignalService - sends signal to peer nodes.
+func (client *peerRESTClient) SignalService(sig serviceSignal) error {
+	values := make(url.Values)
+	values.Set(peerRESTSignal, string(sig))
+	respBody, err := client.call(peerRESTMethodSignalService, values, nil, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+func getRemoteHosts(endpoints EndpointList) []*xnet.Host {
+	var remoteHosts []*xnet.Host
+	for _, hostStr := range GetRemotePeers(endpoints) {
+		host, err := xnet.ParseHost(hostStr)
+		logger.FatalIf(err, "Unable to parse peer Host")
+		remoteHosts = append(remoteHosts, host)
+	}
+
+	return remoteHosts
+}
+
+func getRestClients(peerHosts []*xnet.Host) ([]*peerRESTClient, error) {
+	restClients := make([]*peerRESTClient, len(peerHosts))
+	for i, host := range peerHosts {
+		client, err := newPeerRESTClient(host)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+		}
+		restClients[i] = client
+	}
+
+	return restClients, nil
+}
+
+// Returns a peer rest client.
+func newPeerRESTClient(peer *xnet.Host) (*peerRESTClient, error) {
+
+	scheme := "http"
+	if globalIsSSL {
+		scheme = "https"
+	}
+
+	serverURL := &url.URL{
+		Scheme: scheme,
+		Host:   peer.String(),
+		Path:   peerRESTPath,
+	}
+
+	var tlsConfig *tls.Config
+	if globalIsSSL {
+		tlsConfig = &tls.Config{
+			ServerName: peer.Name,
+			RootCAs:    globalRootCAs,
+		}
+	}
+
+	restClient, err := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
+
+	if err != nil {
+		return &peerRESTClient{host: peer, restClient: restClient, connected: false}, err
+	}
+
+	return &peerRESTClient{host: peer, restClient: restClient, connected: true}, nil
+}

cmd/peer-rest-common.go

@@ -0,0 +1,48 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+const peerRESTVersion = "v1"
+const peerRESTPath = minioReservedBucketPath + "/peer/" + peerRESTVersion
+
+const (
+	peerRESTMethodServerInfo               = "serverinfo"
+	peerRESTMethodCPULoadInfo              = "cpuloadinfo"
+	peerRESTMethodMemUsageInfo             = "memusageinfo"
+	peerRESTMethodDrivePerfInfo            = "driveperfinfo"
+	peerRESTMethodDeleteBucket             = "deletebucket"
+	peerRESTMethodSignalService            = "signalservice"
+	peerRESTMethodGetLocks                 = "getlocks"
+	peerRESTMethodBucketPolicyRemove       = "removebucketpolicy"
+	peerRESTMethodLoadUsers                = "loadusers"
+	peerRESTMethodStartProfiling           = "startprofiling"
+	peerRESTMethodDownloadProfilingData    = "downloadprofilingdata"
+	peerRESTMethodLoadCredentials          = "loadcredentials"
+	peerRESTMethodBucketPolicySet          = "setbucketpolicy"
+	peerRESTMethodBucketNotificationPut    = "putbucketnotification"
+	peerRESTMethodBucketNotificationListen = "listenbucketnotification"
+	peerRESTMethodReloadFormat             = "reloadformat"
+	peerRESTMethodTargetExists             = "targetexists"
+	peerRESTMethodSendEvent                = "sendevent"
+)
+
+const (
+	peerRESTBucket   = "bucket"
+	peerRESTSignal   = "signal"
+	peerRESTProfiler = "profiler"
+	peerRESTDryRun   = "dry-run"
+)

cmd/peer-rest-server.go

@@ -0,0 +1,621 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"net/http"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/event"
+	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/policy"
+)
+
+// To abstract a node over network.
+type peerRESTServer struct {
+}
+
+func getServerInfo() (*ServerInfoData, error) {
+	if globalBootTime.IsZero() {
+		return nil, errServerNotInitialized
+	}
+
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return nil, errServerNotInitialized
+	}
+	// Server info data.
+	return &ServerInfoData{
+		StorageInfo: objLayer.StorageInfo(context.Background()),
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+		Properties: ServerProperties{
+			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			SQSARN:   globalNotificationSys.GetARNList(),
+			Region:   globalServerConfig.GetRegion(),
+		},
+	}, nil
+}
+
+// uptimes - used to sort uptimes in chronological order.
+type uptimes []time.Duration
+
+func (ts uptimes) Len() int {
+	return len(ts)
+}
+
+func (ts uptimes) Less(i, j int) bool {
+	return ts[i] < ts[j]
+}
+
+func (ts uptimes) Swap(i, j int) {
+	ts[i], ts[j] = ts[j], ts[i]
+}
+
+// getPeerUptimes - returns the uptime.
+func getPeerUptimes(serverInfo []ServerInfo) time.Duration {
+	// In a single node Erasure or FS backend setup the uptime of
+	// the setup is the uptime of the single minio server
+	// instance.
+	if !globalIsDistXL {
+		return UTCNow().Sub(globalBootTime)
+	}
+
+	var times []time.Duration
+
+	for _, info := range serverInfo {
+		if info.Error != "" {
+			continue
+		}
+		times = append(times, info.Data.Properties.Uptime)
+	}
+
+	// Sort uptimes in chronological order.
+	sort.Sort(uptimes(times))
+
+	// Return the latest time as the uptime.
+	return times[0]
+}
+
+// GetLocksHandler - returns list of older lock from the server.
+func (s *peerRESTServer) GetLocksHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "GetLocks")
+	locks := globalLockServer.ll.DupLockMap()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(locks))
+
+	w.(http.Flusher).Flush()
+
+}
+
+// LoadUsersHandler - returns server info.
+func (s *peerRESTServer) LoadUsersHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	err := globalIAMSys.Load(objAPI)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// StartProfilingHandler - Issues the start profiling command.
+func (s *peerRESTServer) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	profiler := vars[peerRESTProfiler]
+	if profiler == "" {
+		s.writeErrorResponse(w, errors.New("profiler name is missing"))
+		return
+	}
+
+	if globalProfiler != nil {
+		globalProfiler.Stop()
+	}
+
+	var err error
+	globalProfiler, err = startProfiler(profiler, "")
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// ServerInfoHandler - returns server info.
+func (s *peerRESTServer) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "ServerInfo")
+	info, err := getServerInfo()
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
+// DownloadProflingDataHandler - returns proflied data.
+func (s *peerRESTServer) DownloadProflingDataHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "DownloadProfiling")
+	profileData, err := getProfileData()
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(profileData))
+}
+
+// CPULoadInfoHandler - returns CPU Load info.
+func (s *peerRESTServer) CPULoadInfoHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "CPULoadInfo")
+	info := localEndpointsCPULoad(globalEndpoints)
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
+// DrivePerfInfoHandler - returns Drive Performance info.
+func (s *peerRESTServer) DrivePerfInfoHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "DrivePerfInfo")
+	info := localEndpointsDrivePerf(globalEndpoints)
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
+// MemUsageInfoHandler - returns Memory Usage info.
+func (s *peerRESTServer) MemUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+	ctx := newContext(r, w, "MemUsageInfo")
+	info := localEndpointsMemUsage(globalEndpoints)
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
+// LoadCredentialsHandler - loads credentials.
+func (s *peerRESTServer) LoadCredentialsHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	// Construct path to config.json for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, minioConfigFile)
+	transactionConfigFile := configFile + ".transaction"
+
+	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
+	// and configFile, take a transaction lock to avoid race.
+	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	objLock.RUnlock()
+
+	if err := globalConfigSys.Load(newObjectLayerFn()); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// DeleteBucketHandler - Delete notification and policies related to the bucket.
+func (s *peerRESTServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	if bucketName == "" {
+		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
+		return
+	}
+
+	globalNotificationSys.RemoveNotification(bucketName)
+	globalPolicySys.Remove(bucketName)
+
+	w.(http.Flusher).Flush()
+}
+
+// ReloadFormatHandler - Reload Format.
+func (s *peerRESTServer) ReloadFormatHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	dryRunString := vars[peerRESTDryRun]
+	if dryRunString == "" {
+		s.writeErrorResponse(w, errors.New("dry run parameter is missing"))
+		return
+	}
+
+	var dryRun bool
+	switch strings.ToLower(dryRunString) {
+	case "true":
+		dryRun = true
+	case "false":
+		dryRun = false
+	default:
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+	err := objAPI.ReloadFormat(context.Background(), dryRun)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	w.(http.Flusher).Flush()
+}
+
+// RemoveBucketPolicyHandler - Remove bucket policy.
+func (s *peerRESTServer) RemoveBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	if bucketName == "" {
+		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
+		return
+	}
+
+	globalPolicySys.Remove(bucketName)
+	w.(http.Flusher).Flush()
+}
+
+// SetBucketPolicyHandler - Set bucket policy.
+func (s *peerRESTServer) SetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	if bucketName == "" {
+		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
+		return
+	}
+	var policyData policy.Policy
+	if r.ContentLength < 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&policyData)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	globalPolicySys.Set(bucketName, policyData)
+	w.(http.Flusher).Flush()
+}
+
+type remoteTargetExistsResp struct {
+	Exists bool
+}
+
+// TargetExistsHandler - Check if Target exists.
+func (s *peerRESTServer) TargetExistsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "TargetExists")
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	if bucketName == "" {
+		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
+		return
+	}
+	var targetID event.TargetID
+	if r.ContentLength <= 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&targetID)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	var targetExists remoteTargetExistsResp
+	targetExists.Exists = globalNotificationSys.RemoteTargetExist(bucketName, targetID)
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(&targetExists))
+}
+
+type sendEventRequest struct {
+	Event    event.Event
+	TargetID event.TargetID
+}
+
+type sendEventResp struct {
+	Success bool
+}
+
+// SendEventHandler - Send Event.
+func (s *peerRESTServer) SendEventHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "SendEvent")
+
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	if bucketName == "" {
+		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
+		return
+	}
+	var eventReq sendEventRequest
+	if r.ContentLength <= 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&eventReq)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	var eventResp sendEventResp
+	eventResp.Success = true
+	errs := globalNotificationSys.send(bucketName, eventReq.Event, eventReq.TargetID)
+
+	for i := range errs {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("Event", eventReq.Event.EventName.String())
+		reqInfo.AppendTags("targetName", eventReq.TargetID.Name)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, errs[i].Err)
+
+		eventResp.Success = false
+		s.writeErrorResponse(w, errs[i].Err)
+		return
+	}
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(&eventResp))
+	w.(http.Flusher).Flush()
+}
+
+// PutBucketNotificationHandler - Set bucket policy.
+func (s *peerRESTServer) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	if bucketName == "" {
+		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
+		return
+	}
+
+	var rulesMap event.RulesMap
+	if r.ContentLength < 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&rulesMap)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
+	w.(http.Flusher).Flush()
+}
+
+type listenBucketNotificationReq struct {
+	EventNames []event.Name   `json:"eventNames"`
+	Pattern    string         `json:"pattern"`
+	TargetID   event.TargetID `json:"targetId"`
+	Addr       xnet.Host      `json:"addr"`
+}
+
+// ListenBucketNotificationHandler - Listen bucket notification handler.
+func (s *peerRESTServer) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	if bucketName == "" {
+		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
+		return
+	}
+
+	var args listenBucketNotificationReq
+	if r.ContentLength <= 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&args)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	restClient, err := newPeerRESTClient(&args.Addr)
+	if err != nil {
+		s.writeErrorResponse(w, fmt.Errorf("unable to find PeerRESTClient for provided address %v. This happens only if remote and this minio run with different set of endpoints", args.Addr))
+		return
+	}
+
+	target := NewPeerRESTClientTarget(bucketName, args.TargetID, restClient)
+	rulesMap := event.NewRulesMap(args.EventNames, args.Pattern, target.ID())
+	if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
+		reqInfo := &logger.ReqInfo{BucketName: target.bucketName}
+		reqInfo.AppendTags("target", target.id.Name)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+		s.writeErrorResponse(w, err)
+		return
+	}
+	w.(http.Flusher).Flush()
+}
+
+var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
+
+// SignalServiceHandler - signal service handler.
+func (s *peerRESTServer) SignalServiceHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	signalString := vars[peerRESTSignal]
+	if signalString == "" {
+		s.writeErrorResponse(w, errors.New("signal name is missing"))
+		return
+	}
+	signal := serviceSignal(signalString)
+	defer w.(http.Flusher).Flush()
+	switch signal {
+	case serviceRestart, serviceStop:
+		globalServiceSignalCh <- signal
+	default:
+		s.writeErrorResponse(w, errUnsupportedSignal)
+		return
+	}
+}
+
+func (s *peerRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusForbidden)
+	w.Write([]byte(err.Error()))
+}
+
+// IsValid - To authenticate and verify the time difference.
+func (s *peerRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
+	if err := storageServerRequestValidate(r); err != nil {
+		s.writeErrorResponse(w, err)
+		return false
+	}
+	return true
+}
+
+// registerPeerRESTHandlers - register peer rest router.
+func registerPeerRESTHandlers(router *mux.Router) {
+	server := &peerRESTServer{}
+	subrouter := router.PathPrefix(peerRESTPath).Subrouter()
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodCPULoadInfo).HandlerFunc(httpTraceHdrs(server.CPULoadInfoHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodMemUsageInfo).HandlerFunc(httpTraceHdrs(server.MemUsageInfoHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodDrivePerfInfo).HandlerFunc(httpTraceHdrs(server.DrivePerfInfoHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodDeleteBucket).HandlerFunc(httpTraceHdrs(server.DeleteBucketHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodSignalService).HandlerFunc(httpTraceHdrs(server.SignalServiceHandler)).Queries(restQueries(peerRESTSignal)...)
+
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketPolicyRemove).HandlerFunc(httpTraceAll(server.RemoveBucketPolicyHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketPolicySet).HandlerFunc(httpTraceHdrs(server.SetBucketPolicyHandler)).Queries(restQueries(peerRESTBucket)...)
+
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodLoadUsers).HandlerFunc(httpTraceAll(server.LoadUsersHandler))
+
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodStartProfiling).HandlerFunc(httpTraceAll(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodDownloadProfilingData).HandlerFunc(httpTraceHdrs(server.DownloadProflingDataHandler))
+
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodLoadCredentials).HandlerFunc(httpTraceHdrs(server.LoadCredentialsHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodTargetExists).HandlerFunc(httpTraceHdrs(server.TargetExistsHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodSendEvent).HandlerFunc(httpTraceHdrs(server.SendEventHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketNotificationPut).HandlerFunc(httpTraceHdrs(server.PutBucketNotificationHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketNotificationListen).HandlerFunc(httpTraceHdrs(server.ListenBucketNotificationHandler)).Queries(restQueries(peerRESTBucket)...)
+
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)
+
+	router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
+}

cmd/peer-rpc-client.go

@@ -1,261 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"crypto/tls"
-
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/event"
-	xnet "github.com/minio/minio/pkg/net"
-	"github.com/minio/minio/pkg/policy"
-)
-
-// PeerRPCClient - peer RPC client talks to peer RPC server.
-type PeerRPCClient struct {
-	*RPCClient
-}
-
-// DeleteBucket - calls delete bucket RPC.
-func (rpcClient *PeerRPCClient) DeleteBucket(bucketName string) error {
-	args := DeleteBucketArgs{BucketName: bucketName}
-	reply := VoidReply{}
-	return rpcClient.Call(peerServiceName+".DeleteBucket", &args, &reply)
-}
-
-// SetBucketPolicy - calls set bucket policy RPC.
-func (rpcClient *PeerRPCClient) SetBucketPolicy(bucketName string, bucketPolicy *policy.Policy) error {
-	args := SetBucketPolicyArgs{
-		BucketName: bucketName,
-		Policy:     *bucketPolicy,
-	}
-	reply := VoidReply{}
-	return rpcClient.Call(peerServiceName+".SetBucketPolicy", &args, &reply)
-}
-
-// RemoveBucketPolicy - calls remove bucket policy RPC.
-func (rpcClient *PeerRPCClient) RemoveBucketPolicy(bucketName string) error {
-	args := RemoveBucketPolicyArgs{
-		BucketName: bucketName,
-	}
-	reply := VoidReply{}
-	return rpcClient.Call(peerServiceName+".RemoveBucketPolicy", &args, &reply)
-}
-
-// PutBucketNotification - calls put bukcet notification RPC.
-func (rpcClient *PeerRPCClient) PutBucketNotification(bucketName string, rulesMap event.RulesMap) error {
-	args := PutBucketNotificationArgs{
-		BucketName: bucketName,
-		RulesMap:   rulesMap,
-	}
-	reply := VoidReply{}
-	return rpcClient.Call(peerServiceName+".PutBucketNotification", &args, &reply)
-}
-
-// ListenBucketNotification - calls listen bucket notification RPC.
-func (rpcClient *PeerRPCClient) ListenBucketNotification(bucketName string, eventNames []event.Name,
-	pattern string, targetID event.TargetID, addr xnet.Host) error {
-	args := ListenBucketNotificationArgs{
-		BucketName: bucketName,
-		EventNames: eventNames,
-		Pattern:    pattern,
-		TargetID:   targetID,
-		Addr:       addr,
-	}
-	reply := VoidReply{}
-	return rpcClient.Call(peerServiceName+".ListenBucketNotification", &args, &reply)
-}
-
-// RemoteTargetExist - calls remote target ID exist RPC.
-func (rpcClient *PeerRPCClient) RemoteTargetExist(bucketName string, targetID event.TargetID) (bool, error) {
-	args := RemoteTargetExistArgs{
-		BucketName: bucketName,
-		TargetID:   targetID,
-	}
-	var reply bool
-
-	err := rpcClient.Call(peerServiceName+".RemoteTargetExist", &args, &reply)
-	return reply, err
-}
-
-// SendEvent - calls send event RPC.
-func (rpcClient *PeerRPCClient) SendEvent(bucketName string, targetID, remoteTargetID event.TargetID, eventData event.Event) error {
-	args := SendEventArgs{
-		BucketName: bucketName,
-		TargetID:   remoteTargetID,
-		Event:      eventData,
-	}
-	var reply bool
-
-	err := rpcClient.Call(peerServiceName+".SendEvent", &args, &reply)
-	if err != nil && !reply {
-		reqInfo := &logger.ReqInfo{BucketName: bucketName}
-		reqInfo.AppendTags("targetID", targetID.Name)
-		reqInfo.AppendTags("event", eventData.EventName.String())
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
-		globalNotificationSys.RemoveRemoteTarget(bucketName, targetID)
-	}
-
-	return err
-}
-
-// ReloadFormat - calls reload format RPC.
-func (rpcClient *PeerRPCClient) ReloadFormat(dryRun bool) error {
-	args := ReloadFormatArgs{
-		DryRun: dryRun,
-	}
-	reply := VoidReply{}
-
-	return rpcClient.Call(peerServiceName+".ReloadFormat", &args, &reply)
-}
-
-// LoadUsers - calls load users RPC.
-func (rpcClient *PeerRPCClient) LoadUsers() error {
-	args := AuthArgs{}
-	reply := VoidReply{}
-
-	return rpcClient.Call(peerServiceName+".LoadUsers", &args, &reply)
-}
-
-// LoadCredentials - calls load credentials RPC.
-func (rpcClient *PeerRPCClient) LoadCredentials() error {
-	args := AuthArgs{}
-	reply := VoidReply{}
-
-	return rpcClient.Call(peerServiceName+".LoadCredentials", &args, &reply)
-}
-
-// DrivePerfInfo - returns drive performance info for remote server.
-func (rpcClient *PeerRPCClient) DrivePerfInfo() (ServerDrivesPerfInfo, error) {
-	args := AuthArgs{}
-	var reply ServerDrivesPerfInfo
-
-	err := rpcClient.Call(peerServiceName+".DrivePerfInfo", &args, &reply)
-	return reply, err
-}
-
-// MemUsageInfo - returns mem utilization info for remote server
-func (rpcClient *PeerRPCClient) MemUsageInfo() (ServerMemUsageInfo, error) {
-	args := AuthArgs{}
-	var reply ServerMemUsageInfo
-
-	err := rpcClient.Call(peerServiceName+".MemUsageInfo", &args, &reply)
-	return reply, err
-}
-
-// CPULoadInfo - returns cpu performance info for remote server
-func (rpcClient *PeerRPCClient) CPULoadInfo() (ServerCPULoadInfo, error) {
-	args := AuthArgs{}
-	var reply ServerCPULoadInfo
-
-	err := rpcClient.Call(peerServiceName+".CPULoadInfo", &args, &reply)
-	return reply, err
-}
-
-// StartProfiling - starts profiling on the remote server.
-func (rpcClient *PeerRPCClient) StartProfiling(profiler string) error {
-	args := StartProfilingArgs{Profiler: profiler}
-	reply := VoidReply{}
-	return rpcClient.Call(peerServiceName+".StartProfiling", &args, &reply)
-}
-
-// DownloadProfilingData - download already running profiling on the remote server.
-func (rpcClient *PeerRPCClient) DownloadProfilingData() ([]byte, error) {
-	args := AuthArgs{}
-	var reply []byte
-	err := rpcClient.Call(peerServiceName+".DownloadProfilingData", &args, &reply)
-	return reply, err
-}
-
-// SignalService - calls load server info RPC.
-func (rpcClient *PeerRPCClient) SignalService(sig serviceSignal) error {
-	args := SignalServiceArgs{Sig: sig}
-	reply := VoidReply{}
-
-	return rpcClient.Call(peerServiceName+".SignalService", &args, &reply)
-}
-
-// ServerInfo - calls load server info RPC.
-func (rpcClient *PeerRPCClient) ServerInfo() (ServerInfoData, error) {
-	args := AuthArgs{}
-	reply := ServerInfoData{}
-
-	err := rpcClient.Call(peerServiceName+".ServerInfo", &args, &reply)
-	return reply, err
-}
-
-// GetLocksResp stores various info from the client for each lock that is requested.
-type GetLocksResp map[string][]lockRequesterInfo
-
-// GetLocks - returns the lock information on the server to which the RPC call is made.
-func (rpcClient *PeerRPCClient) GetLocks() (resp GetLocksResp, err error) {
-	err = rpcClient.Call(peerServiceName+".GetLocks", &AuthArgs{}, &resp)
-	return resp, err
-}
-
-// NewPeerRPCClient - returns new peer RPC client.
-func NewPeerRPCClient(host *xnet.Host) (*PeerRPCClient, error) {
-	scheme := "http"
-	if globalIsSSL {
-		scheme = "https"
-	}
-
-	serviceURL := &xnet.URL{
-		Scheme: scheme,
-		Host:   host.String(),
-		Path:   peerServicePath,
-	}
-
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: host.Name,
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	rpcClient, err := NewRPCClient(
-		RPCClientArgs{
-			NewAuthTokenFunc: newAuthToken,
-			RPCVersion:       globalRPCAPIVersion,
-			ServiceName:      peerServiceName,
-			ServiceURL:       serviceURL,
-			TLSConfig:        tlsConfig,
-		},
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return &PeerRPCClient{rpcClient}, nil
-}
-
-// makeRemoteRPCClients - creates Peer RPCClients for given endpoint list.
-func makeRemoteRPCClients(endpoints EndpointList) map[xnet.Host]*PeerRPCClient {
-	peerRPCClientMap := make(map[xnet.Host]*PeerRPCClient)
-	for _, hostStr := range GetRemotePeers(endpoints) {
-		host, err := xnet.ParseHost(hostStr)
-		logger.FatalIf(err, "Unable to parse peer RPC Host")
-		rpcClient, err := NewPeerRPCClient(host)
-		logger.FatalIf(err, "Unable to parse peer RPC Client")
-		peerRPCClientMap[*host] = rpcClient
-	}
-
-	return peerRPCClientMap
-}

cmd/peer-rpc-server.go

@@ -1,419 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"path"
-	"sort"
-	"time"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	xrpc "github.com/minio/minio/cmd/rpc"
-	"github.com/minio/minio/pkg/event"
-	xnet "github.com/minio/minio/pkg/net"
-	"github.com/minio/minio/pkg/policy"
-)
-
-const peerServiceName = "Peer"
-const peerServiceSubPath = "/peer/remote"
-
-var peerServicePath = path.Join(minioReservedBucketPath, peerServiceSubPath)
-
-// peerRPCReceiver - Peer RPC receiver for peer RPC server.
-type peerRPCReceiver struct{}
-
-// DeleteBucketArgs - delete bucket RPC arguments.
-type DeleteBucketArgs struct {
-	AuthArgs
-	BucketName string
-}
-
-// DeleteBucket - handles delete bucket RPC call which removes all values of given bucket in global NotificationSys object.
-func (receiver *peerRPCReceiver) DeleteBucket(args *DeleteBucketArgs, reply *VoidReply) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	globalNotificationSys.RemoveNotification(args.BucketName)
-	globalPolicySys.Remove(args.BucketName)
-	return nil
-}
-
-// SetBucketPolicyArgs - set bucket policy RPC arguments.
-type SetBucketPolicyArgs struct {
-	AuthArgs
-	BucketName string
-	Policy     policy.Policy
-}
-
-// SetBucketPolicy - handles set bucket policy RPC call which adds bucket policy to globalPolicySys.
-func (receiver *peerRPCReceiver) SetBucketPolicy(args *SetBucketPolicyArgs, reply *VoidReply) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	globalPolicySys.Set(args.BucketName, args.Policy)
-	return nil
-}
-
-// RemoveBucketPolicyArgs - delete bucket policy RPC arguments.
-type RemoveBucketPolicyArgs struct {
-	AuthArgs
-	BucketName string
-}
-
-// RemoveBucketPolicy - handles delete bucket policy RPC call which removes bucket policy to globalPolicySys.
-func (receiver *peerRPCReceiver) RemoveBucketPolicy(args *RemoveBucketPolicyArgs, reply *VoidReply) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	globalPolicySys.Remove(args.BucketName)
-	return nil
-}
-
-// PutBucketNotificationArgs - put bucket notification RPC arguments.
-type PutBucketNotificationArgs struct {
-	AuthArgs
-	BucketName string
-	RulesMap   event.RulesMap
-}
-
-// PutBucketNotification - handles put bucket notification RPC call which adds rules to given bucket to global NotificationSys object.
-func (receiver *peerRPCReceiver) PutBucketNotification(args *PutBucketNotificationArgs, reply *VoidReply) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	globalNotificationSys.AddRulesMap(args.BucketName, args.RulesMap)
-	return nil
-}
-
-// ListenBucketNotificationArgs - listen bucket notification RPC arguments.
-type ListenBucketNotificationArgs struct {
-	AuthArgs   `json:"-"`
-	BucketName string         `json:"-"`
-	EventNames []event.Name   `json:"eventNames"`
-	Pattern    string         `json:"pattern"`
-	TargetID   event.TargetID `json:"targetId"`
-	Addr       xnet.Host      `json:"addr"`
-}
-
-// ListenBucketNotification - handles listen bucket notification RPC call.
-// It creates PeerRPCClient target which pushes requested events to target in remote peer.
-func (receiver *peerRPCReceiver) ListenBucketNotification(args *ListenBucketNotificationArgs, reply *VoidReply) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	rpcClient := globalNotificationSys.GetPeerRPCClient(args.Addr)
-	if rpcClient == nil {
-		return fmt.Errorf("unable to find PeerRPCClient for provided address %v. This happens only if remote and this minio run with different set of endpoints", args.Addr)
-	}
-
-	target := NewPeerRPCClientTarget(args.BucketName, args.TargetID, rpcClient)
-	rulesMap := event.NewRulesMap(args.EventNames, args.Pattern, target.ID())
-	if err := globalNotificationSys.AddRemoteTarget(args.BucketName, target, rulesMap); err != nil {
-		reqInfo := &logger.ReqInfo{BucketName: target.bucketName}
-		reqInfo.AppendTags("target", target.id.Name)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
-		return err
-	}
-	return nil
-}
-
-// RemoteTargetExistArgs - remote target ID exist RPC arguments.
-type RemoteTargetExistArgs struct {
-	AuthArgs
-	BucketName string
-	TargetID   event.TargetID
-}
-
-// RemoteTargetExist - handles target ID exist RPC call which checks whether given target ID is a HTTP client target or not.
-func (receiver *peerRPCReceiver) RemoteTargetExist(args *RemoteTargetExistArgs, reply *bool) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	*reply = globalNotificationSys.RemoteTargetExist(args.BucketName, args.TargetID)
-	return nil
-}
-
-// SendEventArgs - send event RPC arguments.
-type SendEventArgs struct {
-	AuthArgs
-	Event      event.Event
-	TargetID   event.TargetID
-	BucketName string
-}
-
-// SendEvent - handles send event RPC call which sends given event to target by given target ID.
-func (receiver *peerRPCReceiver) SendEvent(args *SendEventArgs, reply *bool) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// Set default to true to keep the target.
-	*reply = true
-	errs := globalNotificationSys.send(args.BucketName, args.Event, args.TargetID)
-
-	for i := range errs {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("Event", args.Event.EventName.String())
-		reqInfo.AppendTags("targetName", args.TargetID.Name)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, errs[i].Err)
-
-		*reply = false // send failed i.e. do not keep the target.
-		return errs[i].Err
-	}
-
-	return nil
-}
-
-// ReloadFormatArgs - send event RPC arguments.
-type ReloadFormatArgs struct {
-	AuthArgs
-	DryRun bool
-}
-
-// ReloadFormat - handles reload format RPC call, reloads latest `format.json`
-func (receiver *peerRPCReceiver) ReloadFormat(args *ReloadFormatArgs, reply *VoidReply) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-	return objAPI.ReloadFormat(context.Background(), args.DryRun)
-}
-
-// LoadUsers - handles load users RPC call.
-func (receiver *peerRPCReceiver) LoadUsers(args *AuthArgs, reply *VoidReply) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-	return globalIAMSys.Load(objAPI)
-}
-
-// LoadCredentials - handles load credentials RPC call.
-func (receiver *peerRPCReceiver) LoadCredentials(args *AuthArgs, reply *VoidReply) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// Construct path to config.json for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, minioConfigFile)
-	transactionConfigFile := configFile + ".transaction"
-
-	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
-	// and configFile, take a transaction lock to avoid race.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
-	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	objLock.RUnlock()
-
-	return globalConfigSys.Load(newObjectLayerFn())
-}
-
-// DrivePerfInfo - handles drive performance RPC call
-func (receiver *peerRPCReceiver) DrivePerfInfo(args *AuthArgs, reply *ServerDrivesPerfInfo) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	*reply = localEndpointsDrivePerf(globalEndpoints)
-	return nil
-}
-
-// CPULoadInfo - handles cpu performance RPC call
-func (receiver *peerRPCReceiver) CPULoadInfo(args *AuthArgs, reply *ServerCPULoadInfo) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-	*reply = localEndpointsCPULoad(globalEndpoints)
-	return nil
-}
-
-// MemUsageInfo - handles mem utilization RPC call
-func (receiver *peerRPCReceiver) MemUsageInfo(args *AuthArgs, reply *ServerMemUsageInfo) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-	*reply = localEndpointsMemUsage(globalEndpoints)
-	return nil
-}
-
-// uptimes - used to sort uptimes in chronological order.
-type uptimes []time.Duration
-
-func (ts uptimes) Len() int {
-	return len(ts)
-}
-
-func (ts uptimes) Less(i, j int) bool {
-	return ts[i] < ts[j]
-}
-
-func (ts uptimes) Swap(i, j int) {
-	ts[i], ts[j] = ts[j], ts[i]
-}
-
-// getPeerUptimes - returns the uptime.
-func getPeerUptimes(serverInfo []ServerInfo) time.Duration {
-	// In a single node Erasure or FS backend setup the uptime of
-	// the setup is the uptime of the single minio server
-	// instance.
-	if !globalIsDistXL {
-		return UTCNow().Sub(globalBootTime)
-	}
-
-	var times []time.Duration
-
-	for _, info := range serverInfo {
-		if info.Error != "" {
-			continue
-		}
-		times = append(times, info.Data.Properties.Uptime)
-	}
-
-	// Sort uptimes in chronological order.
-	sort.Sort(uptimes(times))
-
-	// Return the latest time as the uptime.
-	return times[0]
-}
-
-// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
-type StartProfilingArgs struct {
-	AuthArgs
-	Profiler string
-}
-
-// StartProfiling - profiling server receiver.
-func (receiver *peerRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
-	if globalProfiler != nil {
-		globalProfiler.Stop()
-	}
-	var err error
-	globalProfiler, err = startProfiler(args.Profiler, "")
-	return err
-}
-
-// DownloadProfilingData - download profiling data.
-func (receiver *peerRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) error {
-	var err error
-	*reply, err = getProfileData()
-	return err
-}
-
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// SignalServiceArgs - send event RPC arguments.
-type SignalServiceArgs struct {
-	AuthArgs
-	Sig serviceSignal
-}
-
-// SignalService - signal service receiver.
-func (receiver *peerRPCReceiver) SignalService(args *SignalServiceArgs, reply *VoidReply) error {
-	switch args.Sig {
-	case serviceRestart, serviceStop:
-		globalServiceSignalCh <- args.Sig
-	default:
-		return errUnsupportedSignal
-	}
-	return nil
-}
-
-// ServerInfo - server info receiver.
-func (receiver *peerRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) error {
-	if globalBootTime.IsZero() {
-		return errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return errServerNotInitialized
-	}
-
-	// Server info data.
-	*reply = ServerInfoData{
-		StorageInfo: objLayer.StorageInfo(context.Background()),
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			SQSARN:   globalNotificationSys.GetARNList(),
-			Region:   globalServerConfig.GetRegion(),
-		},
-	}
-
-	return nil
-}
-
-// GetLocks - Get Locks receiver.
-func (receiver *peerRPCReceiver) GetLocks(args *AuthArgs, reply *GetLocksResp) error {
-	if globalBootTime.IsZero() {
-		return errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return errServerNotInitialized
-	}
-
-	// Locks data.
-	*reply = globalLockServer.ll.DupLockMap()
-
-	return nil
-}
-
-// NewPeerRPCServer - returns new peer RPC server.
-func NewPeerRPCServer() (*xrpc.Server, error) {
-	rpcServer := xrpc.NewServer()
-	if err := rpcServer.RegisterName(peerServiceName, &peerRPCReceiver{}); err != nil {
-		return nil, err
-	}
-	return rpcServer, nil
-}
-
-// registerPeerRPCRouter - creates and registers Peer RPC server and its router.
-func registerPeerRPCRouter(router *mux.Router) {
-	rpcServer, err := NewPeerRPCServer()
-	logger.FatalIf(err, "Unable to initialize peer RPC Server")
-	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
-	subrouter.Path(peerServiceSubPath).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
-}

cmd/posix-list-dir_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -33,15 +34,27 @@ func TestReadDirFail(t *testing.T) {
 		t.Fatalf("expected = %s, got: %s", errFileNotFound, err)
 	}
 
+	file := path.Join(os.TempDir(), "issue")
+	if err := ioutil.WriteFile(file, []byte(""), 0644); err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(file)
+
 	// Check if file is given.
-	if _, err := readDir("/etc/issue/mydir"); err != errFileNotFound {
+	if _, err := readDir(path.Join(file, "mydir")); err != errFileNotFound {
 		t.Fatalf("expected = %s, got: %s", errFileNotFound, err)
 	}
 
 	// Only valid for linux.
 	if runtime.GOOS == "linux" {
+		permDir := path.Join(os.TempDir(), "perm-dir")
+		if err := os.MkdirAll(permDir, os.FileMode(0200)); err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(permDir)
+
 		// Check if permission denied.
-		if _, err := readDir("/proc/1/fd"); err == nil {
+		if _, err := readDir(permDir); err == nil {
 			t.Fatalf("expected = an error, got: nil")
 		}
 	}

cmd/posix.go

@@ -232,7 +232,6 @@ func getDiskInfo(diskPath string) (di disk.Info, err error) {
 var ignoreDiskFreeOS = []string{
 	globalWindowsOSName,
 	globalNetBSDOSName,
-	globalSolarisOSName,
 }
 
 // check if disk total has minimum required size.
@@ -943,20 +942,6 @@ func (s *posix) openFile(volume, path string, mode int) (f *os.File, err error)
 	return w, nil
 }
 
-// Just like io.LimitedReader but supports Close() to be compatible with io.ReadCloser that is
-// returned by posix.ReadFileStream()
-type posixLimitedReader struct {
-	io.LimitedReader
-}
-
-func (l *posixLimitedReader) Close() error {
-	c, ok := l.R.(io.Closer)
-	if !ok {
-		return errUnexpected
-	}
-	return c.Close()
-}
-
 // ReadFileStream - Returns the read stream of the file.
 func (s *posix) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
 	var err error
@@ -1030,7 +1015,10 @@ func (s *posix) ReadFileStream(volume, path string, offset, length int64) (io.Re
 	if _, err = file.Seek(offset, io.SeekStart); err != nil {
 		return nil, err
 	}
-	return &posixLimitedReader{io.LimitedReader{R: file, N: length}}, nil
+	return struct {
+		io.Reader
+		io.Closer
+	}{Reader: io.LimitReader(file, length), Closer: file}, nil
 }
 
 // CreateFile - creates the file.
@@ -1195,18 +1183,19 @@ func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
 	}
 	st, err := os.Stat((filePath))
 	if err != nil {
-		// File is really not found.
-		if os.IsNotExist(err) {
+		switch {
+		case os.IsNotExist(err):
+			// File is really not found.
 			return FileInfo{}, errFileNotFound
-		} else if isSysErrIO(err) {
+		case isSysErrIO(err):
 			return FileInfo{}, errFaultyDisk
-		} else if isSysErrNotDir(err) {
+		case isSysErrNotDir(err):
 			// File path cannot be verified since one of the parents is a file.
 			return FileInfo{}, errFileNotFound
+		default:
+			// Return all errors here.
+			return FileInfo{}, err
 		}
-
-		// Return all errors here.
-		return FileInfo{}, err
 	}
 	// If its a directory its not a regular file.
 	if st.Mode().IsDir() {
@@ -1231,21 +1220,21 @@ func deleteFile(basePath, deletePath string) error {
 
 	// Attempt to remove path.
 	if err := os.Remove((deletePath)); err != nil {
-		// Ignore errors if the directory is not empty. The server relies on
-		// this functionality, and sometimes uses recursion that should not
-		// error on parent directories.
-		if isSysErrNotEmpty(err) {
+		switch {
+		case isSysErrNotEmpty(err):
+			// Ignore errors if the directory is not empty. The server relies on
+			// this functionality, and sometimes uses recursion that should not
+			// error on parent directories.
 			return nil
-		}
-
-		if os.IsNotExist(err) {
+		case os.IsNotExist(err):
 			return errFileNotFound
-		} else if os.IsPermission(err) {
+		case os.IsPermission(err):
 			return errFileAccessDenied
-		} else if isSysErrIO(err) {
+		case isSysErrIO(err):
 			return errFaultyDisk
+		default:
+			return err
 		}
-		return err
 	}
 
 	// Trailing slash is removed when found to ensure

cmd/posix_test.go

@@ -1069,70 +1069,53 @@ func TestPosixReadFile(t *testing.T) {
 			volume, "path/to/my/object", 0, 5,
 			[]byte("hello"), nil,
 		},
-		// One path segment length is 255 chars long. - 3
-		{
-			volume, "path/to/my/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			0, 5, []byte("hello"), nil},
-		// Whole path is 1024 characters long, success case. - 4
-		{
-			volume, "level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			0, 5, []byte("hello"),
-			func() error {
-				// On darwin HFS does not support > 1024 characters.
-				if runtime.GOOS == "darwin" {
-					return errFileNameTooLong
-				}
-				// On all other platforms return success.
-				return nil
-			}(),
-		},
-		// Object is a directory. - 5
+		// Object is a directory. - 3
 		{
 			volume, "object-as-dir",
 			0, 5, nil, errIsNotRegular},
-		// One path segment length is > 255 chars long. - 6
+		// One path segment length is > 255 chars long. - 4
 		{
 			volume, "path/to/my/object0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
 			0, 5, nil, errFileNameTooLong},
-		// Path length is > 1024 chars long. - 7
+		// Path length is > 1024 chars long. - 5
 		{
 			volume, "level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
 			0, 5, nil, errFileNameTooLong},
-		// Buffer size greater than object size. - 8
+		// Buffer size greater than object size. - 6
 		{
 			volume, "myobject", 0, 16,
 			[]byte("hello, world"),
 			io.ErrUnexpectedEOF,
 		},
-		// Reading from an offset success. - 9
+		// Reading from an offset success. - 7
 		{
 			volume, "myobject", 7, 5,
 			[]byte("world"), nil,
 		},
-		// Reading from an object but buffer size greater. - 10
+		// Reading from an object but buffer size greater. - 8
 		{
 			volume, "myobject",
 			7, 8,
 			[]byte("world"),
 			io.ErrUnexpectedEOF,
 		},
-		// Seeking ahead returns io.EOF. - 11
+		// Seeking ahead returns io.EOF. - 9
 		{
 			volume, "myobject", 14, 1, nil, io.EOF,
 		},
-		// Empty volume name. - 12
+		// Empty volume name. - 10
 		{
 			"", "myobject", 14, 1, nil, errVolumeNotFound,
 		},
-		// Empty filename name. - 13
+		// Empty filename name. - 11
 		{
 			volume, "", 14, 1, nil, errIsNotRegular,
 		},
-		// Non existent volume name - 14.
+		// Non existent volume name - 12
 		{
 			"abcd", "", 14, 1, nil, errVolumeNotFound,
 		},
-		// Non existent filename - 15.
+		// Non existent filename - 13
 		{
 			volume, "abcd", 14, 1, nil, errFileNotFound,
 		},
@@ -1361,8 +1344,7 @@ func TestPosixAppendFile(t *testing.T) {
 		{"myobject", nil},
 		// TestPosix to use same path of previously created file.
 		{"path/to/my/testobject", nil},
-		// One path segment length is 255 chars long.
-		{"path/to/my/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", nil},
+		// TestPosix to use object is a directory now.
 		{"object-as-dir", errIsNotRegular},
 		// path segment uses previously uploaded object.
 		{"myobject/testobject", errFileAccessDenied},
@@ -1372,17 +1354,6 @@ func TestPosixAppendFile(t *testing.T) {
 		{"level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", errFileNameTooLong},
 	}
 
-	// Add path length > 1024 test specially as OS X system does not support 1024 long path.
-	err = errFileNameTooLong
-	if runtime.GOOS != "darwin" {
-		err = nil
-	}
-	// path length is 1024 chars long.
-	testCases = append(testCases, struct {
-		fileName    string
-		expectedErr error
-	}{"level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", err})
-
 	for i, testCase := range testCases {
 		if err = posixStorage.AppendFile("success-vol", testCase.fileName, []byte("hello, world")); err != testCase.expectedErr {
 			t.Errorf("Case: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err)

cmd/post-policy_test.go

@@ -300,7 +300,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 		{
 			objectName:         "test",
 			data:               []byte("Hello, World"),
-			expectedRespStatus: http.StatusBadRequest,
+			expectedRespStatus: http.StatusForbidden,
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			dates:              []interface{}{curTime.Add(-1 * time.Minute * 5).Format(expirationDateFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},
@@ -499,7 +499,7 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t
 // postPresignSignatureV4 - presigned signature for PostPolicy requests.
 func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
 	// Get signining key.
-	signingkey := getSigningKey(secretAccessKey, t, location)
+	signingkey := getSigningKey(secretAccessKey, t, location, "s3")
 	// Calculate signature.
 	signature := getSignature(signingkey, policyBase64)
 	return signature

cmd/postpolicyform.go

@@ -217,10 +217,10 @@ func checkPolicyCond(op string, input1, input2 string) bool {
 
 // checkPostPolicy - apply policy conditions and validate input values.
 // (http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html)
-func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) APIErrorCode {
+func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) error {
 	// Check if policy document expiry date is still not reached
 	if !postPolicyForm.Expiration.After(UTCNow()) {
-		return ErrPolicyAlreadyExpired
+		return fmt.Errorf("Invalid according to Policy: Policy expired")
 	}
 
 	// Flag to indicate if all policies conditions are satisfied
@@ -237,22 +237,23 @@ func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) APIE
 		if startsWithSupported, condFound := startsWithConds[cond]; condFound {
 			// Check if the current condition supports starts-with operator
 			if op == policyCondStartsWith && !startsWithSupported {
-				return ErrAccessDenied
+				return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
 			}
 			// Check if current policy condition is satisfied
-			condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), v.Value)
+			if !condPassed {
+				return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
+			}
 		} else {
 			// This covers all conditions X-Amz-Meta-* and X-Amz-*
 			if strings.HasPrefix(cond, "$x-amz-meta-") || strings.HasPrefix(cond, "$x-amz-") {
 				// Check if policy condition is satisfied
 				condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), v.Value)
+				if !condPassed {
+					return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", op, cond, v.Value)
+				}
 			}
 		}
-		// Check if current policy condition is satisfied, quit immediately otherwise
-		if !condPassed {
-			return ErrAccessDenied
-		}
 	}
 
-	return ErrNone
+	return nil
 }

cmd/postpolicyform_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/base64"
+	"fmt"
 	"net/http"
 	"testing"
 
@@ -45,26 +46,28 @@ func TestPostPolicyForm(t *testing.T) {
 		SuccessActionStatus string
 		Policy              string
 		Expired             bool
-		ErrCode             APIErrorCode
+		expectedErr         error
 	}
 
 	testCases := []testCase{
 		// Everything is fine with this test
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrNone},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: nil},
 		// Expired policy document
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Expired: true, ErrCode: ErrPolicyAlreadyExpired},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Expired: true, expectedErr: fmt.Errorf("Invalid according to Policy: Policy expired")},
 		// Different AMZ date
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "2017T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
 		// Key which doesn't start with user/user1/filename
-		{Bucket: "testbucket", Key: "myfile.txt", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "myfile.txt", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
 		// Incorrect bucket name.
-		{Bucket: "incorrect", Key: "user/user1/filename/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
+		{Bucket: "incorrect", Key: "user/user1/filename/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
 		// Incorrect key name
-		{Bucket: "testbucket", Key: "incorrect", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "incorrect", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
 		// Incorrect date
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "incorrect", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "incorrect", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
 		// Incorrect ContentType
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "incorrect", ErrCode: ErrAccessDenied},
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "incorrect", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
+		// Incorrect Metadata
+		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "151274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed: [eq, $x-amz-meta-uuid, 14365123651274]")},
 	}
 	// Validate all the test cases.
 	for i, tt := range testCases {
@@ -96,8 +99,9 @@ func TestPostPolicyForm(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if errCode := checkPostPolicy(formValues, postPolicyForm); tt.ErrCode != errCode {
-			t.Fatalf("Test %d:, Expected %d, got %d", i+1, tt.ErrCode, errCode)
+		err = checkPostPolicy(formValues, postPolicyForm)
+		if err != nil && tt.expectedErr != nil && err.Error() != tt.expectedErr.Error() {
+			t.Fatalf("Test %d:, Expected %s, got %s", i+1, tt.expectedErr.Error(), err.Error())
 		}
 	}
 }

cmd/routers.go

@@ -38,11 +38,12 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 	// Register storage rpc router only if its a distributed setup.
 	registerStorageRESTHandlers(router, endpoints)
 
+	// Register peer REST router only if its a distributed setup.
+	registerPeerRESTHandlers(router)
+
 	// Register distributed namespace lock.
 	registerDistNSLockRouter(router)
 
-	// Register peer communication router.
-	registerPeerRPCRouter(router)
 }
 
 // List of some generic handlers which are applied for all incoming requests.

cmd/server_test.go

@@ -1693,7 +1693,7 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *check) {
 		{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "url"), []string{"<Key>foo+bar+1</Key>", "<Key>foo+bar+2</Key>"}},
 	}
 
-	for i, testCase := range testCases {
+	for _, testCase := range testCases {
 		// create listObjectsV1 request with valid parameters
 		request, err = newTestSignedRequest("GET", testCase.getURL, 0, nil, s.accessKey, s.secretKey, s.signer)
 		c.Assert(err, nil)
@@ -1706,7 +1706,6 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *check) {
 		getContent, err := ioutil.ReadAll(response.Body)
 		c.Assert(err, nil)
 
-		fmt.Printf("Test %d: %+v vs %+v\n", i+1, string(getContent), testCase.expectedStrings)
 		for _, expectedStr := range testCase.expectedStrings {
 			c.Assert(strings.Contains(string(getContent), expectedStr), true)
 		}

cmd/service.go

@@ -22,12 +22,12 @@ import (
 )
 
 // Type of service signals currently supported.
-type serviceSignal int
+type serviceSignal string
 
 const (
-	serviceStatus  = iota // Gets status about the service.
-	serviceRestart        // Restarts the service.
-	serviceStop           // Stops the server.
+	serviceStatus  serviceSignal = "serviceStatus"  // Gets status about the service.
+	serviceRestart               = "serviceRestart" // Restarts the service.
+	serviceStop                  = "serviceStop"    // Stops the server.
 	// Add new service requests here.
 )
 
@@ -39,7 +39,7 @@ var GlobalServiceDoneCh chan struct{}
 
 // Initialize service mutex once.
 func init() {
-	GlobalServiceDoneCh = make(chan struct{}, 1)
+	GlobalServiceDoneCh = make(chan struct{})
 	globalServiceSignalCh = make(chan serviceSignal)
 }
 

cmd/signals.go

@@ -52,6 +52,9 @@ func handleSignals() {
 		err = globalHTTPServer.Shutdown()
 		logger.LogIf(context.Background(), err)
 
+		// send signal to various go-routines that they need to quit.
+		close(GlobalServiceDoneCh)
+
 		if objAPI := newObjectLayerFn(); objAPI != nil {
 			oerr = objAPI.Shutdown(context.Background())
 			logger.LogIf(context.Background(), oerr)

cmd/signature-v4-parser.go

@@ -47,8 +47,8 @@ func (c credentialHeader) getScope() string {
 	}, "/")
 }
 
-func getReqAccessKeyV4(r *http.Request, region string) (auth.Credentials, bool, APIErrorCode) {
-	ch, err := parseCredentialHeader("Credential="+r.URL.Query().Get("X-Amz-Credential"), region)
+func getReqAccessKeyV4(r *http.Request, region string, stype serviceType) (auth.Credentials, bool, APIErrorCode) {
+	ch, err := parseCredentialHeader("Credential="+r.URL.Query().Get("X-Amz-Credential"), region, stype)
 	if err != ErrNone {
 		// Strip off the Algorithm prefix.
 		v4Auth := strings.TrimPrefix(r.Header.Get("Authorization"), signV4Algorithm)
@@ -56,7 +56,7 @@ func getReqAccessKeyV4(r *http.Request, region string) (auth.Credentials, bool,
 		if len(authFields) != 3 {
 			return auth.Credentials{}, false, ErrMissingFields
 		}
-		ch, err = parseCredentialHeader(authFields[0], region)
+		ch, err = parseCredentialHeader(authFields[0], region, stype)
 		if err != ErrNone {
 			return auth.Credentials{}, false, err
 		}
@@ -65,8 +65,8 @@ func getReqAccessKeyV4(r *http.Request, region string) (auth.Credentials, bool,
 }
 
 // parse credentialHeader string into its structured form.
-func parseCredentialHeader(credElement string, region string) (ch credentialHeader, aec APIErrorCode) {
-	creds := strings.Split(strings.TrimSpace(credElement), "=")
+func parseCredentialHeader(credElement string, region string, stype serviceType) (ch credentialHeader, aec APIErrorCode) {
+	creds := strings.SplitN(strings.TrimSpace(credElement), "=", 2)
 	if len(creds) != 2 {
 		return ch, ErrMissingFields
 	}
@@ -107,7 +107,7 @@ func parseCredentialHeader(credElement string, region string) (ch credentialHead
 		return ch, ErrAuthorizationHeaderMalformed
 
 	}
-	if credElements[2] != "s3" {
+	if credElements[2] != string(stype) {
 		return ch, ErrInvalidService
 	}
 	cred.scope.service = credElements[2]
@@ -185,7 +185,7 @@ func doesV4PresignParamsExist(query url.Values) APIErrorCode {
 }
 
 // Parses all the presigned signature values into separate elements.
-func parsePreSignV4(query url.Values, region string) (psv preSignValues, aec APIErrorCode) {
+func parsePreSignV4(query url.Values, region string, stype serviceType) (psv preSignValues, aec APIErrorCode) {
 	// verify whether the required query params exist.
 	err := doesV4PresignParamsExist(query)
 	if err != ErrNone {
@@ -201,7 +201,7 @@ func parsePreSignV4(query url.Values, region string) (psv preSignValues, aec API
 	preSignV4Values := preSignValues{}
 
 	// Save credential.
-	preSignV4Values.Credential, err = parseCredentialHeader("Credential="+query.Get("X-Amz-Credential"), region)
+	preSignV4Values.Credential, err = parseCredentialHeader("Credential="+query.Get("X-Amz-Credential"), region, stype)
 	if err != ErrNone {
 		return psv, err
 	}
@@ -249,7 +249,7 @@ func parsePreSignV4(query url.Values, region string) (psv preSignValues, aec API
 //    Authorization: algorithm Credential=accessKeyID/credScope, \
 //            SignedHeaders=signedHeaders, Signature=signature
 //
-func parseSignV4(v4Auth string, region string) (sv signValues, aec APIErrorCode) {
+func parseSignV4(v4Auth string, region string, stype serviceType) (sv signValues, aec APIErrorCode) {
 	// Replace all spaced strings, some clients can send spaced
 	// parameters and some won't. So we pro-actively remove any spaces
 	// to make parsing easier.
@@ -275,7 +275,7 @@ func parseSignV4(v4Auth string, region string) (sv signValues, aec APIErrorCode)
 
 	var err APIErrorCode
 	// Save credentail values.
-	signV4Values.Credential, err = parseCredentialHeader(authFields[0], region)
+	signV4Values.Credential, err = parseCredentialHeader(authFields[0], region, stype)
 	if err != ErrNone {
 		return sv, err
 	}

cmd/signature-v4-parser_test.go

@@ -216,10 +216,29 @@ func TestParseCredentialHeader(t *testing.T) {
 				"aws4_request"),
 			expectedErrCode: ErrNone,
 		},
+		// Test Case - 11.
+		// Test case with right inputs -> AccessKey contains `=`. See minio/#7376
+		// "aws4_request" is the valid request version.
+		{
+			inputCredentialStr: generateCredentialStr(
+				"LOCALKEY/DEV/1=",
+				sampleTimeStr,
+				"us-west-1",
+				"s3",
+				"aws4_request"),
+			expectedCredentials: generateCredentials(
+				t,
+				"LOCALKEY/DEV/1=",
+				sampleTimeStr,
+				"us-west-1",
+				"s3",
+				"aws4_request"),
+			expectedErrCode: ErrNone,
+		},
 	}
 
 	for i, testCase := range testCases {
-		actualCredential, actualErrCode := parseCredentialHeader(testCase.inputCredentialStr, "us-west-1")
+		actualCredential, actualErrCode := parseCredentialHeader(testCase.inputCredentialStr, "us-west-1", "s3")
 		// validating the credential fields.
 		if testCase.expectedErrCode != actualErrCode {
 			t.Fatalf("Test %d: Expected the APIErrCode to be %s, got %s", i+1, errorCodes[testCase.expectedErrCode].Code, errorCodes[actualErrCode].Code)
@@ -446,7 +465,7 @@ func TestParseSignV4(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		parsedAuthField, actualErrCode := parseSignV4(testCase.inputV4AuthStr, "")
+		parsedAuthField, actualErrCode := parseSignV4(testCase.inputV4AuthStr, "", "s3")
 
 		if testCase.expectedErrCode != actualErrCode {
 			t.Fatalf("Test %d: Expected the APIErrCode to be %d, got %d", i+1, testCase.expectedErrCode, actualErrCode)
@@ -813,7 +832,7 @@ func TestParsePreSignV4(t *testing.T) {
 			inputQuery.Set(testCase.inputQueryKeyVals[j], testCase.inputQueryKeyVals[j+1])
 		}
 		// call the function under test.
-		parsedPreSign, actualErrCode := parsePreSignV4(inputQuery, "")
+		parsedPreSign, actualErrCode := parsePreSignV4(inputQuery, "", serviceS3)
 		if testCase.expectedErrCode != actualErrCode {
 			t.Fatalf("Test %d: Expected the APIErrCode to be %d, got %d", i+1, testCase.expectedErrCode, actualErrCode)
 		}

cmd/signature-v4-utils.go

@@ -17,11 +17,16 @@
 package cmd
 
 import (
+	"bytes"
+	"context"
 	"crypto/hmac"
+	"encoding/hex"
+	"io/ioutil"
 	"net/http"
 	"strconv"
 	"strings"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/sha256-simd"
 )
@@ -53,7 +58,18 @@ func skipContentSha256Cksum(r *http.Request) bool {
 }
 
 // Returns SHA256 for calculating canonical-request.
-func getContentSha256Cksum(r *http.Request) string {
+func getContentSha256Cksum(r *http.Request, stype serviceType) string {
+	if stype == serviceSTS {
+		payload, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			logger.CriticalIf(context.Background(), err)
+		}
+		sum256 := sha256.New()
+		sum256.Write(payload)
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+		return hex.EncodeToString(sum256.Sum(nil))
+	}
+
 	var (
 		defaultSha256Cksum string
 		v                  []string

cmd/signature-v4-utils_test.go

@@ -248,7 +248,7 @@ func TestGetContentSha256Cksum(t *testing.T) {
 		if testCase.h != "" {
 			r.Header.Set("x-amz-content-sha256", testCase.h)
 		}
-		got := getContentSha256Cksum(r)
+		got := getContentSha256Cksum(r, serviceS3)
 		if got != testCase.expected {
 			t.Errorf("Test %d: got:%s expected:%s", i+1, got, testCase.expected)
 		}

cmd/signature-v4.go

@@ -46,6 +46,13 @@ const (
 	yyyymmdd        = "20060102"
 )
 
+type serviceType string
+
+const (
+	serviceS3  serviceType = "s3"
+	serviceSTS serviceType = "sts"
+)
+
 // getCanonicalHeaders generate a list of request headers with their values
 func getCanonicalHeaders(signedHeaders http.Header) string {
 	var headers []string
@@ -110,7 +117,7 @@ func getScope(t time.Time, region string) string {
 	scope := strings.Join([]string{
 		t.Format(yyyymmdd),
 		region,
-		"s3",
+		string(serviceS3),
 		"aws4_request",
 	}, "/")
 	return scope
@@ -126,10 +133,10 @@ func getStringToSign(canonicalRequest string, t time.Time, scope string) string
 }
 
 // getSigningKey hmac seed to calculate final signature.
-func getSigningKey(secretKey string, t time.Time, region string) []byte {
+func getSigningKey(secretKey string, t time.Time, region string, stype serviceType) []byte {
 	date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format(yyyymmdd)))
 	regionBytes := sumHMAC(date, []byte(region))
-	service := sumHMAC(regionBytes, []byte("s3"))
+	service := sumHMAC(regionBytes, []byte(stype))
 	signingKey := sumHMAC(service, []byte("aws4_request"))
 	return signingKey
 }
@@ -165,7 +172,7 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
 	region := globalServerConfig.GetRegion()
 
 	// Parse credential tag.
-	credHeader, err := parseCredentialHeader("Credential="+formValues.Get("X-Amz-Credential"), region)
+	credHeader, err := parseCredentialHeader("Credential="+formValues.Get("X-Amz-Credential"), region, serviceS3)
 	if err != ErrNone {
 		return ErrMissingFields
 	}
@@ -176,7 +183,7 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
 	}
 
 	// Get signing key.
-	signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region)
+	signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, serviceS3)
 
 	// Get signature.
 	newSignature := getSignature(signingKey, formValues.Get("Policy"))
@@ -193,12 +200,12 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
 // doesPresignedSignatureMatch - Verify query headers with presigned signature
 //     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
 // returns ErrNone if the signature matches.
-func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region string) APIErrorCode {
+func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region string, stype serviceType) APIErrorCode {
 	// Copy request
 	req := *r
 
 	// Parse request query string.
-	pSignValues, err := parsePreSignV4(req.URL.Query(), region)
+	pSignValues, err := parsePreSignV4(req.URL.Query(), region, stype)
 	if err != ErrNone {
 		return err
 	}
@@ -240,7 +247,7 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	query.Set("X-Amz-Date", t.Format(iso8601Format))
 	query.Set("X-Amz-Expires", strconv.Itoa(expireSeconds))
 	query.Set("X-Amz-SignedHeaders", getSignedHeaders(extractedSignedHeaders))
-	query.Set("X-Amz-Credential", cred.AccessKey+"/"+getScope(t, pSignValues.Credential.scope.region))
+	query.Set("X-Amz-Credential", cred.AccessKey+"/"+pSignValues.Credential.getScope())
 
 	// Save other headers available in the request parameters.
 	for k, v := range req.URL.Query() {
@@ -291,7 +298,8 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	presignedStringToSign := getStringToSign(presignedCanonicalReq, t, pSignValues.Credential.getScope())
 
 	// Get hmac presigned signing key.
-	presignedSigningKey := getSigningKey(cred.SecretKey, pSignValues.Credential.scope.date, pSignValues.Credential.scope.region)
+	presignedSigningKey := getSigningKey(cred.SecretKey, pSignValues.Credential.scope.date,
+		pSignValues.Credential.scope.region, stype)
 
 	// Get new signature.
 	newSignature := getSignature(presignedSigningKey, presignedStringToSign)
@@ -306,7 +314,7 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 // doesSignatureMatch - Verify authorization header with calculated header in accordance with
 //     - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
 // returns ErrNone if signature matches.
-func doesSignatureMatch(hashedPayload string, r *http.Request, region string) APIErrorCode {
+func doesSignatureMatch(hashedPayload string, r *http.Request, region string, stype serviceType) APIErrorCode {
 	// Copy request.
 	req := *r
 
@@ -314,7 +322,7 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string) AP
 	v4Auth := req.Header.Get("Authorization")
 
 	// Parse signature version '4' header.
-	signV4Values, err := parseSignV4(v4Auth, region)
+	signV4Values, err := parseSignV4(v4Auth, region, stype)
 	if err != ErrNone {
 		return err
 	}
@@ -354,7 +362,8 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string) AP
 	stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope())
 
 	// Get hmac signing key.
-	signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, signV4Values.Credential.scope.region)
+	signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date,
+		signV4Values.Credential.scope.region, stype)
 
 	// Calculate signature.
 	newSignature := getSignature(signingKey, stringToSign)

cmd/signature-v4_test.go

@@ -74,7 +74,7 @@ func TestDoesPolicySignatureMatch(t *testing.T) {
 				"X-Amz-Date": []string{now.Format(iso8601Format)},
 				"X-Amz-Signature": []string{
 					getSignature(getSigningKey(globalServerConfig.GetCredential().SecretKey, now,
-						globalMinioDefaultRegion), "policy"),
+						globalMinioDefaultRegion, serviceS3), "policy"),
 				},
 				"Policy": []string{"policy"},
 			},
@@ -293,7 +293,7 @@ func TestDoesPresignedSignatureMatch(t *testing.T) {
 		}
 
 		// Check if it matches!
-		err := doesPresignedSignatureMatch(payloadSHA256, req, testCase.region)
+		err := doesPresignedSignatureMatch(payloadSHA256, req, testCase.region, serviceS3)
 		if err != testCase.expected {
 			t.Errorf("(%d) expected to get %s, instead got %s", i, niceError(testCase.expected), niceError(err))
 		}

cmd/storage-errors.go

@@ -66,9 +66,12 @@ var errVolumeNotEmpty = errors.New("volume is not empty")
 // errVolumeAccessDenied - cannot access volume, insufficient permissions.
 var errVolumeAccessDenied = errors.New("volume access denied")
 
-// errVolumeAccessDenied - cannot access file, insufficient permissions.
+// errFileAccessDenied - cannot access file, insufficient permissions.
 var errFileAccessDenied = errors.New("file access denied")
 
+// errFileParentIsFile - cannot have overlapping objects, parent is already a file.
+var errFileParentIsFile = errors.New("parent is a file")
+
 // errBitrotHashAlgoInvalid - the algo for bit-rot hash
 // verification is empty or invalid.
 var errBitrotHashAlgoInvalid = errors.New("bit-rot hash algorithm is invalid")

cmd/storage-rest-client.go

@@ -25,7 +25,6 @@ import (
 	"net/url"
 	"path"
 	"strconv"
-	"time"
 
 	"encoding/gob"
 	"encoding/hex"
@@ -38,10 +37,6 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )
 
-// The timeout of TCP connect and sending/receiving
-// data for all internode storage REST requests.
-const storageRESTTimeout = 5 * time.Minute
-
 func isNetworkError(err error) bool {
 	if err == nil {
 		return false
@@ -405,7 +400,7 @@ func newStorageRESTClient(endpoint Endpoint) (*storageRESTClient, error) {
 		}
 	}
 
-	restClient, err := rest.NewClient(serverURL, tlsConfig, storageRESTTimeout, newAuthToken)
+	restClient, err := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
 	if err != nil {
 		return nil, err
 	}

cmd/storage-rest-server.go

@@ -17,17 +17,14 @@
 package cmd
 
 import (
+	"encoding/gob"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"path"
 	"strconv"
-
-	"net/http"
-
-	"encoding/gob"
-	"encoding/hex"
-
 	"time"
 
 	"github.com/gorilla/mux"
@@ -47,6 +44,7 @@ type storageRESTServer struct {
 func (s *storageRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
 	w.WriteHeader(http.StatusForbidden)
 	w.Write([]byte(err.Error()))
+	w.(http.Flusher).Flush()
 }
 
 // Authenticates storage client's requests and validates for skewed time.
@@ -93,6 +91,7 @@ func (s *storageRESTServer) GetInstanceID(w http.ResponseWriter, r *http.Request
 	}
 	w.Header().Set("Content-Length", strconv.Itoa(len(s.instanceID)))
 	w.Write([]byte(s.instanceID))
+	w.(http.Flusher).Flush()
 }
 
 // DiskInfoHandler - returns disk info.
@@ -268,6 +267,7 @@ func (s *storageRESTServer) ReadAllHandler(w http.ResponseWriter, r *http.Reques
 	}
 	w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
 	w.Write(buf)
+	w.(http.Flusher).Flush()
 }
 
 // ReadFileHandler - read section of a file.
@@ -311,6 +311,7 @@ func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Reque
 	}
 	w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
 	w.Write(buf)
+	w.(http.Flusher).Flush()
 }
 
 // ReadFileHandler - read section of a file.
@@ -331,6 +332,7 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
 		s.writeErrorResponse(w, err)
 		return
 	}
+
 	rc, err := s.storage.ReadFileStream(volume, filePath, int64(offset), int64(length))
 	if err != nil {
 		s.writeErrorResponse(w, err)
@@ -338,7 +340,9 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
 	}
 	defer rc.Close()
 	w.Header().Set("Content-Length", strconv.Itoa(length))
+
 	io.Copy(w, rc)
+	w.(http.Flusher).Flush()
 }
 
 // ListDirHandler - list a directory.

cmd/streaming-signature-v4.go

@@ -52,7 +52,7 @@ func getChunkSignature(cred auth.Credentials, seedSignature string, region strin
 		hashedChunk
 
 	// Get hmac signing key.
-	signingKey := getSigningKey(cred.SecretKey, date, region)
+	signingKey := getSigningKey(cred.SecretKey, date, region, serviceS3)
 
 	// Calculate signature.
 	newSignature := getSignature(signingKey, stringToSign)
@@ -72,7 +72,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 	v4Auth := req.Header.Get("Authorization")
 
 	// Parse signature version '4' header.
-	signV4Values, errCode := parseSignV4(v4Auth, globalServerConfig.GetRegion())
+	signV4Values, errCode := parseSignV4(v4Auth, globalServerConfig.GetRegion(), serviceS3)
 	if errCode != ErrNone {
 		return cred, "", "", time.Time{}, errCode
 	}
@@ -124,7 +124,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 	stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope())
 
 	// Get hmac signing key.
-	signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region)
+	signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, serviceS3)
 
 	// Calculate signature.
 	newSignature := getSignature(signingKey, stringToSign)

cmd/sts-datatypes.go

@@ -42,6 +42,39 @@ type AssumedRoleUser struct {
 	// contains filtered or unexported fields
 }
 
+// AssumeRoleResponse contains the result of successful AssumeRole request.
+type AssumeRoleResponse struct {
+	XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleResponse" json:"-"`
+
+	Result           AssumeRoleResult `xml:"AssumeRoleResult"`
+	ResponseMetadata struct {
+		RequestID string `xml:"RequestId,omitempty"`
+	} `xml:"ResponseMetadata,omitempty"`
+}
+
+// AssumeRoleResult - Contains the response to a successful AssumeRole
+// request, including temporary credentials that can be used to make
+// Minio API requests.
+type AssumeRoleResult struct {
+	// The identifiers for the temporary security credentials that the operation
+	// returns.
+	AssumedRoleUser AssumedRoleUser `xml:",omitempty"`
+
+	// The temporary security credentials, which include an access key ID, a secret
+	// access key, and a security (or session) token.
+	//
+	// Note: The size of the security token that STS APIs return is not fixed. We
+	// strongly recommend that you make no assumptions about the maximum size. As
+	// of this writing, the typical size is less than 4096 bytes, but that can vary.
+	// Also, future updates to AWS might require larger sizes.
+	Credentials auth.Credentials `xml:",omitempty"`
+
+	// A percentage value that indicates the size of the policy in packed form.
+	// The service rejects any policy with a packed size greater than 100 percent,
+	// which means the policy exceeded the allowed space.
+	PackedPolicySize int `xml:",omitempty"`
+}
+
 // AssumeRoleWithWebIdentityResponse contains the result of successful AssumeRoleWithWebIdentity request.
 type AssumeRoleWithWebIdentityResponse struct {
 	XMLName          xml.Name          `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithWebIdentityResponse" json:"-"`

cmd/sts-errors.go

@@ -53,6 +53,7 @@ type STSErrorCode int
 // Error codes, non exhaustive list - http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithSAML.html
 const (
 	ErrSTSNone STSErrorCode = iota
+	ErrSTSAccessDenied
 	ErrSTSMissingParameter
 	ErrSTSInvalidParameterValue
 	ErrSTSWebIdentityExpiredToken
@@ -76,6 +77,11 @@ func (e stsErrorCodeMap) ToSTSErr(errCode STSErrorCode) STSError {
 // error code to STSError structure, these fields carry respective
 // descriptions for all the error responses.
 var stsErrCodes = stsErrorCodeMap{
+	ErrSTSAccessDenied: {
+		Code:           "AccessDenied",
+		Description:    "Generating temporary credentials not allowed for this request.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	ErrSTSMissingParameter: {
 		Code:           "MissingParameter",
 		Description:    "A required parameter for the specified action is not supplied.",

cmd/sts-handlers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 
@@ -33,6 +34,7 @@ const (
 	// STS API action constants
 	clientGrants = "AssumeRoleWithClientGrants"
 	webIdentity  = "AssumeRoleWithWebIdentity"
+	assumeRole   = "AssumeRole"
 )
 
 // stsAPIHandlers implements and provides http handlers for AWS STS API.
@@ -46,6 +48,11 @@ func registerSTSRouter(router *mux.Router) {
 	// STS Router
 	stsRouter := router.NewRoute().PathPrefix("/").Subrouter()
 
+	// Assume roles with no JWT, handles AssumeRole.
+	stsRouter.Methods("POST").HeadersRegexp("Content-Type", "application/x-www-form-urlencoded*").
+		HeadersRegexp("Authorization", "AWS4-HMAC-SHA256*").
+		HandlerFunc(httpTraceAll(sts.AssumeRole))
+
 	// Assume roles with JWT handler, handles both ClientGrants and WebIdentity.
 	stsRouter.Methods("POST").HeadersRegexp("Content-Type", "application/x-www-form-urlencoded*").
 		HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))
@@ -64,8 +71,136 @@ func registerSTSRouter(router *mux.Router) {
 
 }
 
+func checkAssumeRoleAuth(ctx context.Context, r *http.Request) (user auth.Credentials, stsErr STSErrorCode) {
+	switch getRequestAuthType(r) {
+	default:
+		return user, ErrSTSAccessDenied
+	case authTypeSigned:
+		s3Err := isReqAuthenticated(ctx, r, globalServerConfig.GetRegion(), serviceSTS)
+		if STSErrorCode(s3Err) != ErrSTSNone {
+			return user, STSErrorCode(s3Err)
+		}
+		var owner bool
+		user, owner, s3Err = getReqAccessKeyV4(r, globalServerConfig.GetRegion(), serviceSTS)
+		if STSErrorCode(s3Err) != ErrSTSNone {
+			return user, STSErrorCode(s3Err)
+		}
+		// Root credentials are not allowed to use STS API
+		if owner {
+			return user, ErrSTSAccessDenied
+		}
+	}
+
+	// Session tokens are not allowed in STS AssumeRole requests.
+	if getSessionToken(r) != "" {
+		return user, ErrSTSAccessDenied
+	}
+
+	return user, ErrSTSNone
+}
+
+// AssumeRole - implementation of AWS STS API AssumeRole to get temporary
+// credentials for regular users on Minio.
+// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
+func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AssumeRole")
+
+	user, stsErr := checkAssumeRoleAuth(ctx, r)
+	if stsErr != ErrSTSNone {
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(stsErr))
+		return
+	}
+
+	if err := r.ParseForm(); err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	if r.Form.Get("Policy") != "" {
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	if r.Form.Get("Version") != stsAPIVersion {
+		logger.LogIf(ctx, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSMissingParameter))
+		return
+	}
+
+	action := r.Form.Get("Action")
+	switch action {
+	case assumeRole:
+	default:
+		logger.LogIf(ctx, fmt.Errorf("Unsupported action %s", action))
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	ctx = newContext(r, w, action)
+	defer logger.AuditLog(w, r, action, nil)
+
+	var err error
+	m := make(map[string]interface{})
+	m["exp"], err = validator.GetDefaultExpiration(r.Form.Get("DurationSeconds"))
+	if err != nil {
+		switch err {
+		case validator.ErrInvalidDuration:
+			writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		default:
+			logger.LogIf(ctx, err)
+			writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		}
+		return
+	}
+
+	policyName, err := globalIAMSys.GetUserPolicy(user.AccessKey)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	// This policy is the policy associated with the user
+	// requesting for temporary credentials. The temporary
+	// credentials will inherit the same policy requirements.
+	m["policy"] = policyName
+
+	secret := globalServerConfig.GetCredential().SecretKey
+	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInternalError))
+		return
+	}
+
+	// Set the newly generated credentials.
+	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInternalError))
+		return
+	}
+
+	// Notify all other Minio peers to reload temp users
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	assumeRoleResponse := &AssumeRoleResponse{
+		Result: AssumeRoleResult{
+			Credentials: cred,
+		},
+	}
+
+	assumeRoleResponse.ResponseMetadata.RequestID = w.Header().Get(responseRequestIDKey)
+	writeSuccessResponseXML(w, encodeResponse(assumeRoleResponse))
+}
+
 func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AssumeRoleInternalFunction")
+	ctx := newContext(r, w, "AssumeRoleJWTCommon")
 
 	// Parse the incoming form data.
 	if err := r.ParseForm(); err != nil {
@@ -74,6 +209,11 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	if r.Form.Get("Policy") != "" {
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
 	if r.Form.Get("Version") != stsAPIVersion {
 		logger.LogIf(ctx, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
 		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSMissingParameter))
@@ -169,19 +309,23 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 	var encodedSuccessResponse []byte
 	switch action {
 	case clientGrants:
-		encodedSuccessResponse = encodeResponse(&AssumeRoleWithClientGrantsResponse{
+		clientGrantsResponse := &AssumeRoleWithClientGrantsResponse{
 			Result: ClientGrantsResult{
 				Credentials:      cred,
 				SubjectFromToken: subFromToken,
 			},
-		})
+		}
+		clientGrantsResponse.ResponseMetadata.RequestID = w.Header().Get(responseRequestIDKey)
+		encodedSuccessResponse = encodeResponse(clientGrantsResponse)
 	case webIdentity:
-		encodedSuccessResponse = encodeResponse(&AssumeRoleWithWebIdentityResponse{
+		webIdentityResponse := &AssumeRoleWithWebIdentityResponse{
 			Result: WebIdentityResult{
 				Credentials:                 cred,
 				SubjectFromWebIdentityToken: subFromToken,
 			},
-		})
+		}
+		webIdentityResponse.ResponseMetadata.RequestID = w.Header().Get(responseRequestIDKey)
+		encodedSuccessResponse = encodeResponse(webIdentityResponse)
 	}
 
 	writeSuccessResponseXML(w, encodedSuccessResponse)

cmd/test-utils_test.go

@@ -407,6 +407,22 @@ func resetGlobalConfigPath() {
 	globalConfigDir = &ConfigDir{path: ""}
 }
 
+func resetGlobalServiceDoneCh() {
+	// Repeatedly send on the service done channel, so that
+	// listening go-routines will quit. This works better than
+	// closing the channel - closing introduces a new race, as the
+	// current thread writes to the variable, and other threads
+	// listening on it, read from it.
+loop:
+	for {
+		select {
+		case GlobalServiceDoneCh <- struct{}{}:
+		default:
+			break loop
+		}
+	}
+}
+
 // sets globalObjectAPI to `nil`.
 func resetGlobalObjectAPI() {
 	globalObjLayerMutex.Lock()
@@ -479,6 +495,9 @@ func resetGlobalIAMSys() {
 // Resets all the globals used modified in tests.
 // Resetting ensures that the changes made to globals by one test doesn't affect others.
 func resetTestGlobals() {
+	// close any indefinitely running go-routines from previous
+	// tests.
+	resetGlobalServiceDoneCh()
 	// set globalObjectAPI to `nil`.
 	resetGlobalObjectAPI()
 	// Reset config path set.
@@ -680,7 +699,7 @@ func signStreamingRequest(req *http.Request, accessKey, secretKey string, currTi
 	scope := strings.Join([]string{
 		currTime.Format(yyyymmdd),
 		globalMinioDefaultRegion,
-		"s3",
+		string(serviceS3),
 		"aws4_request",
 	}, "/")
 
@@ -690,7 +709,7 @@ func signStreamingRequest(req *http.Request, accessKey, secretKey string, currTi
 
 	date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd)))
 	region := sumHMAC(date, []byte(globalMinioDefaultRegion))
-	service := sumHMAC(region, []byte("s3"))
+	service := sumHMAC(region, []byte(string(serviceS3)))
 	signingKey := sumHMAC(service, []byte("aws4_request"))
 
 	signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
@@ -760,7 +779,7 @@ func assembleStreamingChunks(req *http.Request, body io.ReadSeeker, chunkSize in
 		scope := strings.Join([]string{
 			currTime.Format(yyyymmdd),
 			regionStr,
-			"s3",
+			string(serviceS3),
 			"aws4_request",
 		}, "/")
 
@@ -773,7 +792,7 @@ func assembleStreamingChunks(req *http.Request, body io.ReadSeeker, chunkSize in
 
 		date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd)))
 		region := sumHMAC(date, []byte(regionStr))
-		service := sumHMAC(region, []byte("s3"))
+		service := sumHMAC(region, []byte(serviceS3))
 		signingKey := sumHMAC(service, []byte("aws4_request"))
 
 		signature = hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
@@ -874,7 +893,7 @@ func preSignV4(req *http.Request, accessKeyID, secretAccessKey string, expires i
 	queryStr := strings.Replace(query.Encode(), "+", "%20", -1)
 	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method)
 	stringToSign := getStringToSign(canonicalRequest, date, scope)
-	signingKey := getSigningKey(secretAccessKey, date, region)
+	signingKey := getSigningKey(secretAccessKey, date, region, serviceS3)
 	signature := getSignature(signingKey, stringToSign)
 
 	req.URL.RawQuery = query.Encode()
@@ -1035,7 +1054,7 @@ func signRequestV4(req *http.Request, accessKey, secretKey string) error {
 	scope := strings.Join([]string{
 		currTime.Format(yyyymmdd),
 		region,
-		"s3",
+		string(serviceS3),
 		"aws4_request",
 	}, "/")
 
@@ -1045,7 +1064,7 @@ func signRequestV4(req *http.Request, accessKey, secretKey string) error {
 
 	date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd)))
 	regionHMAC := sumHMAC(date, []byte(region))
-	service := sumHMAC(regionHMAC, []byte("s3"))
+	service := sumHMAC(regionHMAC, []byte(serviceS3))
 	signingKey := sumHMAC(service, []byte("aws4_request"))
 
 	signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))

cmd/tree-walk-pool.go

@@ -33,7 +33,6 @@ type listParams struct {
 	recursive bool
 	marker    string
 	prefix    string
-	heal      bool
 }
 
 // errWalkAbort - returned by doTreeWalk() if it returns prematurely.

cmd/update-main_test.go

@@ -23,6 +23,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 )
@@ -82,13 +83,19 @@ func TestReleaseTagToNFromTimeConversion(t *testing.T) {
 func TestDownloadURL(t *testing.T) {
 	minioVersion1 := releaseTimeToReleaseTag(UTCNow())
 	durl := getDownloadURL(minioVersion1)
-	if runtime.GOOS == "windows" {
-		if durl != minioReleaseURL+"minio.exe" {
-			t.Errorf("Expected %s, got %s", minioReleaseURL+"minio.exe", durl)
+	if IsDocker() {
+		if durl != "docker pull minio/minio:"+minioVersion1 {
+			t.Errorf("Expected %s, got %s", "docker pull minio/minio:"+minioVersion1, durl)
 		}
 	} else {
-		if durl != minioReleaseURL+"minio" {
-			t.Errorf("Expected %s, got %s", minioReleaseURL+"minio", durl)
+		if runtime.GOOS == "windows" {
+			if durl != minioReleaseURL+"minio.exe" {
+				t.Errorf("Expected %s, got %s", minioReleaseURL+"minio.exe", durl)
+			}
+		} else {
+			if durl != minioReleaseURL+"minio" {
+				t.Errorf("Expected %s, got %s", minioReleaseURL+"minio", durl)
+			}
 		}
 	}
 
@@ -141,8 +148,12 @@ func TestUserAgent(t *testing.T) {
 			os.Setenv("MARATHON_APP_LABEL_DCOS_PACKAGE_VERSION", "mesos-1111")
 		}
 		str := getUserAgent(testCase.mode)
-		if str != testCase.expectedStr {
-			t.Errorf("Test %d: expected: %s, got: %s", i+1, testCase.expectedStr, str)
+		expectedStr := testCase.expectedStr
+		if IsDocker() {
+			expectedStr = strings.Replace(expectedStr, "; source", "; docker; source", -1)
+		}
+		if str != expectedStr {
+			t.Errorf("Test %d: expected: %s, got: %s", i+1, expectedStr, str)
 		}
 		os.Unsetenv("MARATHON_APP_LABEL_DCOS_PACKAGE_VERSION")
 		os.Unsetenv(testCase.envName)

cmd/web-handlers.go

@@ -39,6 +39,7 @@ import (
 	"github.com/gorilla/rpc/v2/json2"
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/s3utils"
+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/browser"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
@@ -47,7 +48,7 @@ import (
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -157,7 +158,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
 	}
 
 	// Check if bucket is a reserved bucket name or invalid.
-	if isReservedOrInvalidBucket(args.BucketName) {
+	if isReservedOrInvalidBucket(args.BucketName, true) {
 		return toJSONError(errInvalidBucketName)
 	}
 
@@ -296,24 +297,29 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
 	// If etcd, dns federation configured list buckets from etcd.
 	if globalDNSConfig != nil {
 		dnsBuckets, err := globalDNSConfig.List()
-		if err != nil {
+		if err != nil && err != dns.ErrNoEntriesFound {
 			return toJSONError(err)
 		}
+		bucketSet := set.NewStringSet()
 		for _, dnsRecord := range dnsBuckets {
-			bucketName := strings.Trim(dnsRecord.Key, "/")
+			if bucketSet.Contains(dnsRecord.Key) {
+				continue
+			}
 
 			if globalIAMSys.IsAllowed(iampolicy.Args{
 				AccountName:     claims.Subject,
 				Action:          iampolicy.ListBucketAction,
-				BucketName:      bucketName,
+				BucketName:      dnsRecord.Key,
 				ConditionValues: getConditionValues(r, "", claims.Subject),
 				IsOwner:         owner,
 				ObjectName:      "",
 			}) {
 				reply.Buckets = append(reply.Buckets, WebBucketInfo{
-					Name:         bucketName,
+					Name:         dnsRecord.Key,
 					CreationDate: dnsRecord.CreationDate,
 				})
+
+				bucketSet.Add(dnsRecord.Key)
 			}
 		}
 	} else {
@@ -912,26 +918,15 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
 		metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
 
-		pipeReader, pipeWriter := io.Pipe()
-		snappyWriter := snappy.NewBufferedWriter(pipeWriter)
-
-		var actualReader *hash.Reader
-		actualReader, err = hash.NewReader(reader, size, "", "", actualSize)
+		actualReader, err := hash.NewReader(reader, size, "", "", actualSize)
 		if err != nil {
 			writeWebErrorResponse(w, err)
 			return
 		}
 
-		go func() {
-			// Writing to the compressed writer.
-			_, cerr := io.CopyN(snappyWriter, actualReader, actualSize)
-			snappyWriter.Close()
-			pipeWriter.CloseWithError(cerr)
-		}()
-
 		// Set compression metrics.
 		size = -1 // Since compressed size is un-predictable.
-		reader = pipeReader
+		reader = newSnappyCompressReader(actualReader)
 		hashReader, err = hash.NewReader(reader, size, "", "", actualSize)
 		if err != nil {
 			writeWebErrorResponse(w, err)
@@ -1753,7 +1748,7 @@ func presignedGet(host, bucket, object string, expiry int64, creds auth.Credenti
 	extractedSignedHeaders.Set("host", host)
 	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, path, "GET")
 	stringToSign := getStringToSign(canonicalRequest, date, getScope(date, region))
-	signingKey := getSigningKey(secretKey, date, region)
+	signingKey := getSigningKey(secretKey, date, region, serviceS3)
 	signature := getSignature(signingKey, stringToSign)
 
 	// Construct the final presigned URL.
@@ -1803,67 +1798,50 @@ func toJSONError(err error, params ...string) (jerr *json2.Error) {
 
 // toWebAPIError - convert into error into APIError.
 func toWebAPIError(err error) APIError {
-	if err == errAuthentication {
-		return APIError{
-			Code:           "AccessDenied",
-			HTTPStatusCode: http.StatusForbidden,
-			Description:    err.Error(),
-		}
-	} else if err == errServerNotInitialized {
+	switch err {
+	case errServerNotInitialized:
 		return APIError{
 			Code:           "XMinioServerNotInitialized",
 			HTTPStatusCode: http.StatusServiceUnavailable,
 			Description:    err.Error(),
 		}
-	} else if err == auth.ErrInvalidAccessKeyLength {
+	case errAuthentication, auth.ErrInvalidAccessKeyLength, auth.ErrInvalidSecretKeyLength, errInvalidAccessKeyID:
 		return APIError{
 			Code:           "AccessDenied",
 			HTTPStatusCode: http.StatusForbidden,
 			Description:    err.Error(),
 		}
-	} else if err == auth.ErrInvalidSecretKeyLength {
-		return APIError{
-			Code:           "AccessDenied",
-			HTTPStatusCode: http.StatusForbidden,
-			Description:    err.Error(),
-		}
-	} else if err == errInvalidAccessKeyID {
-		return APIError{
-			Code:           "AccessDenied",
-			HTTPStatusCode: http.StatusForbidden,
-			Description:    err.Error(),
-		}
-	} else if err == errSizeUnspecified {
+	case errSizeUnspecified:
 		return APIError{
 			Code:           "InvalidRequest",
 			HTTPStatusCode: http.StatusBadRequest,
 			Description:    err.Error(),
 		}
-	} else if err == errChangeCredNotAllowed {
+	case errChangeCredNotAllowed:
 		return APIError{
 			Code:           "MethodNotAllowed",
 			HTTPStatusCode: http.StatusMethodNotAllowed,
 			Description:    err.Error(),
 		}
-	} else if err == errInvalidBucketName {
+	case errInvalidBucketName:
 		return APIError{
 			Code:           "InvalidBucketName",
 			HTTPStatusCode: http.StatusBadRequest,
 			Description:    err.Error(),
 		}
-	} else if err == errInvalidArgument {
+	case errInvalidArgument:
 		return APIError{
 			Code:           "InvalidArgument",
 			HTTPStatusCode: http.StatusBadRequest,
 			Description:    err.Error(),
 		}
-	} else if err == errEncryptedObject {
+	case errEncryptedObject:
 		return getAPIError(ErrSSEEncryptedObject)
-	} else if err == errInvalidEncryptionParameters {
+	case errInvalidEncryptionParameters:
 		return getAPIError(ErrInvalidEncryptionParameters)
-	} else if err == errObjectTampered {
+	case errObjectTampered:
 		return getAPIError(ErrObjectTampered)
-	} else if err == errMethodNotAllowed {
+	case errMethodNotAllowed:
 		return getAPIError(ErrMethodNotAllowed)
 	}
 

cmd/xl-sets.go

@@ -641,22 +641,27 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir(). Sets passes set of disks.
-func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, sets ...[]StorageAPI) listDirFunc {
+func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, sets ...*xlObjects) listDirFunc {
 	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string) {
-		for _, disk := range disks {
+		var diskEntries = make([][]string, len(disks))
+		var wg sync.WaitGroup
+		for index, disk := range disks {
 			if disk == nil {
 				continue
 			}
+			wg.Add(1)
+			go func(index int, disk StorageAPI) {
+				defer wg.Done()
+				diskEntries[index], _ = disk.ListDir(bucket, prefixDir, -1)
+			}(index, disk)
+		}
 
-			var entries []string
+		wg.Wait()
+
+		// Find elements in entries which are not in mergedEntries
+		for _, entries := range diskEntries {
 			var newEntries []string
-			var err error
-			entries, err = disk.ListDir(bucket, prefixDir, -1)
-			if err != nil {
-				continue
-			}
 
-			// Find elements in entries which are not in mergedEntries
 			for _, entry := range entries {
 				idx := sort.SearchStrings(mergedEntries, entry)
 				// if entry is already present in mergedEntries don't add.
@@ -672,16 +677,16 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 				sort.Strings(mergedEntries)
 			}
 		}
+
 		return mergedEntries
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
 	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
-		for _, disks := range sets {
-			entries := listDirInternal(bucket, prefixDir, prefixEntry, disks)
+		for _, set := range sets {
 			var newEntries []string
 			// Find elements in entries which are not in mergedEntries
-			for _, entry := range entries {
+			for _, entry := range listDirInternal(bucket, prefixDir, prefixEntry, set.getLoadBalancedDisks()) {
 				idx := sort.SearchStrings(mergedEntries, entry)
 				// if entry is already present in mergedEntries don't add.
 				if idx < len(mergedEntries) && mergedEntries[idx] == entry {
@@ -696,8 +701,7 @@ func listDirSetsFactory(ctx context.Context, isLeaf isLeafFunc, isLeafDir isLeaf
 				sort.Strings(mergedEntries)
 			}
 		}
-		mergedEntries, delayIsLeaf = filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
-		return mergedEntries, delayIsLeaf
+		return filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
 	}
 	return listDir
 }
@@ -721,7 +725,7 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
 		recursive = false
 	}
 
-	walkResultCh, endWalkCh := s.listPool.Release(listParams{bucket, recursive, marker, prefix, false})
+	walkResultCh, endWalkCh := s.listPool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
 		isLeaf := func(bucket, entry string) bool {
@@ -743,12 +747,7 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
 			return false
 		}
 
-		var setDisks = make([][]StorageAPI, len(s.sets))
-		for _, set := range s.sets {
-			setDisks = append(setDisks, set.getLoadBalancedDisks())
-		}
-
-		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, setDisks...)
+		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, s.sets...)
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 
@@ -799,7 +798,7 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
 		}
 	}
 
-	params := listParams{bucket, recursive, nextMarker, prefix, false}
+	params := listParams{bucket, recursive, nextMarker, prefix}
 	if !eof {
 		s.listPool.Set(params, walkResultCh, endWalkCh)
 	}
@@ -1011,9 +1010,6 @@ func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
 // If it is a single node XL and all disks are root disks, it is most likely a test setup, else it is a production setup.
 // On a test setup we allow creation of format.json on root disks to help with dev/testing.
 func isTestSetup(infos []DiskInfo, errs []error) bool {
-	if globalIsDistXL {
-		return false
-	}
 	rootDiskCount := 0
 	for i := range errs {
 		if errs[i] != nil {
@@ -1300,8 +1296,8 @@ func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun, remove b
 }
 
 // HealObject - heals inconsistent object on a hashedSet based on object name.
-func (s *xlSets) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (madmin.HealResultItem, error) {
-	return s.getHashedSet(object).HealObject(ctx, bucket, object, dryRun, remove)
+func (s *xlSets) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (madmin.HealResultItem, error) {
+	return s.getHashedSet(object).HealObject(ctx, bucket, object, dryRun, remove, scanMode)
 }
 
 // Lists all buckets which need healing.
@@ -1323,115 +1319,48 @@ func (s *xlSets) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
 	return listBuckets, nil
 }
 
-// listObjectsHeal - wrapper function implemented over file tree walk.
-func (s *xlSets) listObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	// Default is recursive, if delimiter is set then list non recursive.
+// HealObjects - Heal all objects recursively at a specified prefix, any
+// dangling objects deleted as well automatically.
+func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, healObjectFn func(string, string) error) (err error) {
 	recursive := true
-	if delimiter == slashSeparator {
-		recursive = false
+
+	endWalkCh := make(chan struct{})
+	isLeaf := func(bucket, entry string) bool {
+		entry = strings.TrimSuffix(entry, slashSeparator)
+		// Verify if we are at the leaf, a leaf is where we
+		// see `xl.json` inside a directory.
+		return s.getHashedSet(entry).isObject(bucket, entry)
 	}
 
-	// "heal" true for listObjectsHeal() and false for listObjects()
-	walkResultCh, endWalkCh := s.listPool.Release(listParams{bucket, recursive, marker, prefix, true})
-	if walkResultCh == nil {
-		endWalkCh = make(chan struct{})
-		isLeaf := func(bucket, entry string) bool {
-			entry = strings.TrimSuffix(entry, slashSeparator)
-			// Verify if we are at the leaf, a leaf is where we
-			// see `xl.json` inside a directory.
-			return s.getHashedSet(entry).isObject(bucket, entry)
-		}
-
-		isLeafDir := func(bucket, entry string) bool {
-			var ok bool
-			for _, set := range s.sets {
-				ok = set.isObjectDir(bucket, entry)
-				if ok {
-					return true
-				}
-			}
-			return false
-		}
-
-		var setDisks = make([][]StorageAPI, len(s.sets))
+	isLeafDir := func(bucket, entry string) bool {
+		var ok bool
 		for _, set := range s.sets {
-			setDisks = append(setDisks, set.getLoadBalancedDisks())
+			ok = set.isObjectDir(bucket, entry)
+			if ok {
+				return true
+			}
 		}
-
-		listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, setDisks...)
-		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
+		return false
 	}
 
-	var objInfos []ObjectInfo
-	var eof bool
-	var nextMarker string
-	for i := 0; i < maxKeys; {
+	listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, s.sets...)
+	walkResultCh := startTreeWalk(ctx, bucket, prefix, "", recursive, listDir, isLeaf, isLeafDir, endWalkCh)
+	for {
 		walkResult, ok := <-walkResultCh
 		if !ok {
-			// Closed channel.
-			eof = true
 			break
 		}
 		// For any walk error return right away.
 		if walkResult.err != nil {
-			return loi, toObjectErr(walkResult.err, bucket, prefix)
+			return toObjectErr(walkResult.err, bucket, prefix)
+		}
+		if err := healObjectFn(bucket, walkResult.entry); err != nil {
+			return toObjectErr(err, bucket, walkResult.entry)
 		}
-		var objInfo ObjectInfo
-		objInfo.Bucket = bucket
-		objInfo.Name = walkResult.entry
-		nextMarker = objInfo.Name
-		objInfos = append(objInfos, objInfo)
-		i++
 		if walkResult.end {
-			eof = true
 			break
 		}
 	}
 
-	params := listParams{bucket, recursive, nextMarker, prefix, true}
-	if !eof {
-		s.listPool.Set(params, walkResultCh, endWalkCh)
-	}
-
-	result := ListObjectsInfo{IsTruncated: !eof}
-	for _, objInfo := range objInfos {
-		result.NextMarker = objInfo.Name
-		result.Objects = append(result.Objects, objInfo)
-	}
-	return result, nil
-}
-
-// This is not implemented yet, will be implemented later to comply with Admin API refactor.
-func (s *xlSets) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
-	if err = checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, s); err != nil {
-		return loi, err
-	}
-
-	// With max keys of zero we have reached eof, return right here.
-	if maxKeys == 0 {
-		return loi, nil
-	}
-
-	// For delimiter and prefix as '/' we do not list anything at all
-	// since according to s3 spec we stop at the 'delimiter' along
-	// with the prefix. On a flat namespace with 'prefix' as '/'
-	// we don't have any entries, since all the keys are of form 'keyName/...'
-	if delimiter == slashSeparator && prefix == slashSeparator {
-		return loi, nil
-	}
-
-	// Over flowing count - reset to maxObjectList.
-	if maxKeys < 0 || maxKeys > maxObjectList {
-		maxKeys = maxObjectList
-	}
-
-	// Initiate a list operation, if successful filter and return quickly.
-	listObjInfo, err := s.listObjectsHeal(ctx, bucket, prefix, marker, delimiter, maxKeys)
-	if err == nil {
-		// We got the entries successfully return.
-		return listObjInfo, nil
-	}
-
-	// Return error at the end.
-	return loi, toObjectErr(err, bucket, prefix)
+	return nil
 }

cmd/xl-v1-bucket.go

@@ -21,6 +21,7 @@ import (
 	"sort"
 	"sync"
 
+	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -36,8 +37,7 @@ var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
 // MakeBucket - make a bucket.
 func (xl xlObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
 	// Verify if bucket is valid.
-	if !IsValidBucketName(bucket) {
-		logger.LogIf(ctx, BucketNameInvalid{Bucket: bucket})
+	if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
 		return BucketNameInvalid{Bucket: bucket}
 	}
 
@@ -178,7 +178,7 @@ func (xl xlObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketInfo,
 			// should take care of this.
 			var bucketsInfo []BucketInfo
 			for _, volInfo := range volsInfo {
-				if isReservedOrInvalidBucket(volInfo.Name) {
+				if isReservedOrInvalidBucket(volInfo.Name, true) {
 					continue
 				}
 				bucketsInfo = append(bucketsInfo, BucketInfo(volInfo))

cmd/xl-v1-healing-common.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // commonTime returns a maximally occurring time from a list of time.
@@ -158,7 +159,7 @@ func getLatestXLMeta(ctx context.Context, partsMetadata []xlMetaV1, errs []error
 // - slice of errors about the state of data files on disk - can have
 //   a not-found error or a hash-mismatch error.
 func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs []error, bucket,
-	object string) ([]StorageAPI, []error) {
+	object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) {
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
 
@@ -168,27 +169,38 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			continue
 		}
 
-		erasureInfo := partsMetadata[i].Erasure
-		erasure, err := NewErasure(ctx, erasureInfo.DataBlocks, erasureInfo.ParityBlocks, erasureInfo.BlockSize)
-		if err != nil {
-			dataErrs[i] = err
-			continue
-		}
-
-		// disk has a valid xl.json but may not have all the
-		// parts. This is considered an outdated disk, since
-		// it needs healing too.
-		for _, part := range partsMetadata[i].Parts {
-			checksumInfo := erasureInfo.GetChecksumInfo(part.Name)
-			tillOffset := erasure.ShardFileTillOffset(0, part.Size, part.Size)
-			err = bitrotCheckFile(onlineDisk, bucket, pathJoin(object, part.Name), tillOffset, checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
+		switch scanMode {
+		case madmin.HealDeepScan:
+			erasureInfo := partsMetadata[i].Erasure
+			erasure, err := NewErasure(ctx, erasureInfo.DataBlocks, erasureInfo.ParityBlocks, erasureInfo.BlockSize)
 			if err != nil {
-				isCorrupt := strings.HasPrefix(err.Error(), "Bitrot verification mismatch - expected ")
-				if !isCorrupt && err != errFileNotFound && err != errVolumeNotFound {
-					logger.LogIf(ctx, err)
-				}
 				dataErrs[i] = err
-				break
+				continue
+			}
+
+			// disk has a valid xl.json but may not have all the
+			// parts. This is considered an outdated disk, since
+			// it needs healing too.
+			for _, part := range partsMetadata[i].Parts {
+				checksumInfo := erasureInfo.GetChecksumInfo(part.Name)
+				tillOffset := erasure.ShardFileTillOffset(0, part.Size, part.Size)
+				err = bitrotCheckFile(onlineDisk, bucket, pathJoin(object, part.Name), tillOffset, checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
+				if err != nil {
+					isCorrupt := strings.HasPrefix(err.Error(), "Bitrot verification mismatch - expected ")
+					if !isCorrupt && err != errFileNotFound && err != errVolumeNotFound {
+						logger.LogIf(ctx, err)
+					}
+					dataErrs[i] = err
+					break
+				}
+			}
+		case madmin.HealNormalScan:
+			for _, part := range partsMetadata[i].Parts {
+				_, err := onlineDisk.StatFile(bucket, pathJoin(object, part.Name))
+				if err != nil {
+					dataErrs[i] = err
+					break
+				}
 			}
 		}
 

cmd/xl-v1-healing-common_test.go

@@ -23,6 +23,8 @@ import (
 	"path/filepath"
 	"testing"
 	"time"
+
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // validates functionality provided to find most common
@@ -239,7 +241,7 @@ func TestListOnlineDisks(t *testing.T) {
 				i+1, test.expectedTime, modTime)
 		}
 
-		availableDisks, newErrs := disksWithAllParts(context.Background(), onlineDisks, partsMetadata, test.errs, bucket, object)
+		availableDisks, newErrs := disksWithAllParts(context.Background(), onlineDisks, partsMetadata, test.errs, bucket, object, madmin.HealDeepScan)
 		test.errs = newErrs
 
 		if test._tamperBackend != noTamper {
@@ -291,7 +293,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to read xl meta data %v", err)
 	}
 
-	filteredDisks, errs := disksWithAllParts(ctx, xlDisks, partsMetadata, errs, bucket, object)
+	filteredDisks, errs := disksWithAllParts(ctx, xlDisks, partsMetadata, errs, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(xlDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -328,7 +330,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	}
 
 	errs = make([]error, len(xlDisks))
-	filteredDisks, errs = disksWithAllParts(ctx, xlDisks, partsMetadata, errs, bucket, object)
+	filteredDisks, errs = disksWithAllParts(ctx, xlDisks, partsMetadata, errs, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(xlDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))

cmd/xl-v1-healing.go

@@ -208,7 +208,8 @@ func shouldHealObjectOnDisk(xlErr, dataErr error, meta xlMetaV1, quorumModTime t
 
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func healObject(ctx context.Context, storageDisks []StorageAPI, bucket string, object string,
-	quorum int, dryRun bool) (result madmin.HealResultItem, err error) {
+	quorum int, dryRun bool, scanMode madmin.HealScanMode) (result madmin.HealResultItem, err error) {
+
 	partsMetadata, errs := readAllXLMetadata(ctx, storageDisks, bucket, object)
 
 	errCount := 0
@@ -232,7 +233,7 @@ func healObject(ctx context.Context, storageDisks []StorageAPI, bucket string, o
 	latestDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
 	// List of disks having all parts as per latest xl.json.
-	availableDisks, dataErrs := disksWithAllParts(ctx, latestDisks, partsMetadata, errs, bucket, object)
+	availableDisks, dataErrs := disksWithAllParts(ctx, latestDisks, partsMetadata, errs, bucket, object, scanMode)
 
 	// Initialize heal result object
 	result = madmin.HealResultItem{
@@ -621,7 +622,7 @@ func (xl xlObjects) isObjectDangling(metaArr []xlMetaV1, errs []error) (validMet
 // FIXME: If an object object was deleted and one disk was down,
 // and later the disk comes back up again, heal on the object
 // should delete it.
-func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool, remove bool) (hr madmin.HealResultItem, err error) {
+func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool, remove bool, scanMode madmin.HealScanMode) (hr madmin.HealResultItem, err error) {
 	// Create context that also contains information about the object and bucket.
 	// The top level handler might not have this information.
 	reqInfo := logger.GetReqInfo(ctx)
@@ -670,5 +671,5 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 	defer objectLock.RUnlock()
 
 	// Heal the object.
-	return healObject(healCtx, xl.getDisks(), bucket, object, latestXLMeta.Erasure.DataBlocks, dryRun)
+	return healObject(healCtx, xl.getDisks(), bucket, object, latestXLMeta.Erasure.DataBlocks, dryRun, scanMode)
 }

cmd/xl-v1-healing_test.go

@@ -21,6 +21,8 @@ import (
 	"context"
 	"path/filepath"
 	"testing"
+
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // Tests undoes and validates if the undoing completes successfully.
@@ -114,7 +116,7 @@ func TestHealObjectXL(t *testing.T) {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
 
-	_, err = obj.HealObject(context.Background(), bucket, object, false, false)
+	_, err = obj.HealObject(context.Background(), bucket, object, false, false, madmin.HealNormalScan)
 	if err != nil {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
@@ -130,7 +132,7 @@ func TestHealObjectXL(t *testing.T) {
 	}
 
 	// Try healing now, expect to receive errDiskNotFound.
-	_, err = obj.HealObject(context.Background(), bucket, object, false, false)
+	_, err = obj.HealObject(context.Background(), bucket, object, false, false, madmin.HealDeepScan)
 	// since majority of xl.jsons are not available, object quorum can't be read properly and error will be errXLReadQuorum
 	if _, ok := err.(InsufficientReadQuorum); !ok {
 		t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err)

cmd/xl-v1-list-objects-heal.go

@@ -23,7 +23,7 @@ func (xl xlObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
 	return nil, nil
 }
 
-// This is not implemented/needed anymore, look for xl-sets.ListObjectsHeal()
-func (xl xlObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	return loi, nil
+// This is not implemented/needed anymore, look for xl-sets.HealObjects()
+func (xl xlObjects) HealObjects(ctx context.Context, bucket, prefix string, healFn func(string, string) error) (e error) {
+	return nil
 }

cmd/xl-v1-list-objects.go

@@ -69,8 +69,7 @@ func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, del
 		recursive = false
 	}
 
-	heal := false // true only for xl.ListObjectsHeal
-	walkResultCh, endWalkCh := xl.listPool.Release(listParams{bucket, recursive, marker, prefix, heal})
+	walkResultCh, endWalkCh := xl.listPool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
 		isLeaf := xl.isObject
@@ -125,7 +124,7 @@ func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, del
 		}
 	}
 
-	params := listParams{bucket, recursive, nextMarker, prefix, heal}
+	params := listParams{bucket, recursive, nextMarker, prefix}
 	if !eof {
 		xl.listPool.Set(params, walkResultCh, endWalkCh)
 	}

cmd/xl-v1-metadata.go

@@ -21,6 +21,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"net/http"
 	"path"
 	"sort"
 	"sync"
@@ -226,7 +227,16 @@ func (m xlMetaV1) ToObjectInfo(bucket, object string) ObjectInfo {
 		ContentType:     m.Meta["content-type"],
 		ContentEncoding: m.Meta["content-encoding"],
 	}
-
+	// Update expires
+	var (
+		t time.Time
+		e error
+	)
+	if exp, ok := m.Meta["expires"]; ok {
+		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+			objInfo.Expires = t.UTC()
+		}
+	}
 	objInfo.backendType = BackendErasure
 
 	// Extract etag from metadata.

cmd/xl-v1-multipart.go

@@ -624,7 +624,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	// Check if an object is present as one of the parent dir.
 	// -- FIXME. (needs a new kind of lock).
 	if xl.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-		return oi, toObjectErr(errFileAccessDenied, bucket, object)
+		return oi, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
 	// Calculate s3 compatible md5sum for complete multipart.

cmd/xl-v1-object.go

@@ -182,7 +182,7 @@ func (xl xlObjects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 			nsUnlocker()
 			return nil, toObjectErr(err, bucket, object)
 		}
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, nsUnlocker), nil
+		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
 	}
 
 	var objInfo ObjectInfo
@@ -192,7 +192,7 @@ func (xl xlObjects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 		return nil, toObjectErr(err, bucket, object)
 	}
 
-	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, nsUnlocker)
+	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
 	if nErr != nil {
 		return nil, nErr
 	}
@@ -206,7 +206,7 @@ func (xl xlObjects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 	// case of incomplete read.
 	pipeCloser := func() { pr.Close() }
 
-	return fn(pr, h, pipeCloser)
+	return fn(pr, h, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject - reads an object erasured coded across multiple
@@ -577,7 +577,7 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 		// -- FIXME. (needs a new kind of lock).
 		// -- FIXME (this also causes performance issue when disks are down).
 		if xl.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+			return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 		}
 
 		if err = xl.putObjectDir(ctx, minioMetaTmpBucket, tempObj, writeQuorum); err != nil {
@@ -608,7 +608,7 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 	// -- FIXME. (needs a new kind of lock).
 	// -- FIXME (this also causes performance issue when disks are down).
 	if xl.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-		return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
 	// Limit the reader to its provided size if specified.

cmd/xl-v1-object_test.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 func TestRepeatPutObjectPart(t *testing.T) {
@@ -308,7 +309,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = xl.HealObject(context.Background(), bucket, object, false, false)
+	_, err = xl.HealObject(context.Background(), bucket, object, false, false, madmin.HealNormalScan)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -337,7 +338,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = xl.HealObject(context.Background(), bucket, object, false, false)
+	_, err = xl.HealObject(context.Background(), bucket, object, false, false, madmin.HealDeepScan)
 	if err != nil {
 		t.Fatal(err)
 	}

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
  minio1:
-  image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+  image: minio/minio:RELEASE.2019-03-13T21-59-47Z
   volumes:
    - data1:/data
   ports:
@@ -15,7 +15,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio2:
-  image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+  image: minio/minio:RELEASE.2019-03-13T21-59-47Z
   volumes:
    - data2:/data
   ports:
@@ -25,7 +25,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio3:
-  image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+  image: minio/minio:RELEASE.2019-03-13T21-59-47Z
   volumes:
    - data3:/data
   ports:
@@ -35,7 +35,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio4:
-  image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+  image: minio/minio:RELEASE.2019-03-13T21-59-47Z
   volumes:
    - data4:/data
   ports:

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+    image: minio/minio:RELEASE.2019-03-13T21-59-47Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
       - access_key
 
   minio2:
-    image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+    image: minio/minio:RELEASE.2019-03-13T21-59-47Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
       - access_key
 
   minio3:
-    image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+    image: minio/minio:RELEASE.2019-03-13T21-59-47Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
       - access_key
 
   minio4:
-    image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+    image: minio/minio:RELEASE.2019-03-13T21-59-47Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+    image: minio/minio:RELEASE.2019-03-13T21-59-47Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio2:
-    image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+    image: minio/minio:RELEASE.2019-03-13T21-59-47Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio3:
-    image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+    image: minio/minio:RELEASE.2019-03-13T21-59-47Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio4:
-    image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+    image: minio/minio:RELEASE.2019-03-13T21-59-47Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/kubernetes/README.md

@@ -197,7 +197,7 @@ kubectl delete statefulset minio \
 
 If your cluster does not have a storage solution or PV abstraction, you must explicitly define what nodes you wish to run Minio on, and define a homogeneous path to a local fast block device available on every host.
 
-This must be changed in the example daemonset: [minio-distributed-daemonset.yaml](minio-distributed-daemonset.yaml)
+This must be changed in the example daemonset: [minio-distributed-daemonset.yaml](https://github.com/minio/minio/blob/master/docs/orchestration/kubernetes/minio-distributed-daemonset.yaml)
 
 Specifically the hostpath:
 ```yaml

docs/orchestration/kubernetes/minio-distributed-daemonset.yaml

@@ -30,7 +30,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+        image: minio/minio:RELEASE.2019-03-13T21-59-47Z
         # Unfortunately you must manually define each server. Perhaps autodiscovery via DNS can be implemented in the future.
         args:
         - server

docs/orchestration/kubernetes/minio-distributed-statefulset.yaml

@@ -21,7 +21,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+        image: minio/minio:RELEASE.2019-03-13T21-59-47Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes/minio-gcs-gateway-deployment.yaml

@@ -21,7 +21,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+        image: minio/minio:RELEASE.2019-03-13T21-59-47Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-standalone-deployment.yaml

@@ -29,7 +29,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-02-20T22-44-29Z
+        image: minio/minio:RELEASE.2019-03-13T21-59-47Z
         args:
         - server
         - /data