Revert "cache: Rewrite to cache only on download (#7575)" (#7684)

This reverts commit a13b58f630.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,46 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: community, triage
+assignees: ''
+
+---
+
+<!--- Provide a general summary of the issue in the Title above -->
+
+## Expected Behavior
+<!--- If you're describing a bug, tell us what should happen -->
+<!--- If you're suggesting a change/improvement, tell us how it should work -->
+
+## Current Behavior
+<!--- If describing a bug, tell us what happens instead of the expected behavior -->
+<!--- If suggesting a change/improvement, explain the difference from current behavior -->
+
+## Possible Solution
+<!--- Not obligatory, but suggest a fix/reason for the bug, -->
+<!--- or ideas how to implement the addition or change -->
+
+## Steps to Reproduce (for bugs)
+<!--- Provide a link to a live example, or an unambiguous set of steps to -->
+<!--- reproduce this bug. Include code to reproduce, if relevant -->
+1.
+2.
+3.
+4.
+
+## Context
+<!--- How has this issue affected you? What are you trying to accomplish? -->
+<!--- Providing context helps us come up with a solution that is most useful in the real world -->
+
+## Regression
+<!-- Is this issue a regression? (Yes / No) -->
+<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
+
+## Your Environment
+<!--- Include as many relevant details about the environment you experienced the bug in -->
+* Version used (`minio version`):
+* Environment name and version (e.g. nginx 1.9.1):
+* Server type and version:
+* Operating System and version (`uname -a`):
+* Link to your project:

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: community, triage
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# MinIO Contribution Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+# MinIO Contribution Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
 
 ``MinIO`` community welcomes your contribution. To make the process as seamless as possible, we recommend you read this contribution guide.
 

Makefile

@@ -74,7 +74,7 @@ docker: build
 # Builds minio and installs it to $GOPATH/bin.
 install: build
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
-	@mkdir -p $(GOPATH)/bin && cp -uf $(PWD)/minio $(GOPATH)/bin/minio
+	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
 clean:

README.md

@@ -1,5 +1,5 @@
 # MinIO Quickstart Guide
-[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
 
 MinIO is an object storage server released under Apache License v2.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
 

README_zh_CN.md

@@ -1,4 +1,4 @@
-# MinIO Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+# MinIO Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
 
 MinIO 是一个基于Apache License v2.0开源协议的对象存储服务。它兼容亚马逊S3云存储服务接口，非常适合于存储大容量非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。
 

browser/app/js/browser/Login.js

@@ -48,7 +48,7 @@ export class Login extends React.Component {
 
   handleSubmit(event) {
     event.preventDefault()
-    const { showAlert, history } = this.props
+    const { showAlert, clearAlert, history } = this.props
     let message = ""
     if (this.state.accessKey === "") {
       message = "Access Key cannot be empty"
@@ -66,6 +66,9 @@ export class Login extends React.Component {
         password: this.state.secretKey
       })
       .then(res => {
+        // Clear alerts from previous login attempts
+        clearAlert()
+
         history.push("/")
       })
       .catch(e => {
@@ -145,4 +148,7 @@ const mapDispatchToProps = dispatch => {
   }
 }
 
-export default connect(state => state, mapDispatchToProps)(Login)
+export default connect(
+  state => state,
+  mapDispatchToProps
+)(Login)

browser/app/js/objects/actions.js

@@ -208,7 +208,8 @@ export const shareObject = (object, days, hours, minutes) => {
         .PresignedGet({
           host: location.host,
           bucket: currentBucket,
-          object: objectName
+          object: objectName,
+          expiry: expiry
         })
         .then(obj => {
           dispatch(showShareObject(object, obj.url))

cmd/admin-handlers.go

@@ -35,7 +35,6 @@ import (
 	"github.com/tidwall/sjson"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/handlers"
@@ -1416,70 +1415,3 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	// Send success response
 	writeSuccessResponseHeadersOnly(w)
 }
-
-// UpdateAdminCredsHandler - POST /minio/admin/v1/config/credential
-// ----------
-// Update admin credentials in a minio server
-func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
-	r *http.Request) {
-
-	ctx := newContext(r, w, "UpdateCredentialsHandler")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	// Avoid setting new credentials when they are already passed
-	// by the environment. Deny if WORM is enabled.
-	if globalIsEnvCreds || globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
-		// More than maxConfigSize bytes were available
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
-		return
-	}
-
-	password := globalServerConfig.GetCredential().SecretKey
-	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
-		return
-	}
-
-	// Decode request body
-	var req madmin.SetCredsReq
-	if err = json.Unmarshal(configBytes, &req); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrRequestBodyParse), r.URL)
-		return
-	}
-
-	creds, err := auth.CreateCredentials(req.AccessKey, req.SecretKey)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Acquire lock before updating global configuration.
-	globalServerConfigMu.Lock()
-	defer globalServerConfigMu.Unlock()
-
-	// Update local credentials in memory.
-	globalServerConfig.SetCredential(creds)
-
-	// Set active creds.
-	globalActiveCred = creds
-
-	if err = saveServerConfig(ctx, objectAPI, globalServerConfig); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Reply to the client before restarting minio server.
-	writeSuccessResponseHeadersOnly(w)
-}

cmd/admin-handlers_test.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -298,60 +297,6 @@ func (atb *adminXLTestBed) TearDown() {
 	resetTestGlobals()
 }
 
-func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
-	// Create an object myobject under bucket mybucket.
-	bucketName := "mybucket"
-	err := atb.objLayer.MakeBucketWithLocation(context.Background(), bucketName, "")
-	if err != nil {
-		t.Fatalf("Failed to make bucket %s - %v", bucketName,
-			err)
-	}
-
-	// create some objects
-	{
-		objName := "myobject"
-		for i := 0; i < 10; i++ {
-			objectName := fmt.Sprintf("%s-%d", objName, i)
-			_, err = atb.objLayer.PutObject(context.Background(), bucketName, objectName,
-				mustGetPutObjReader(t, bytes.NewReader([]byte("hello")),
-					int64(len("hello")), "", ""), ObjectOptions{})
-			if err != nil {
-				t.Fatalf("Failed to create %s - %v", objectName,
-					err)
-			}
-		}
-	}
-
-	// create a multipart upload (incomplete)
-	{
-		objName := "mpObject"
-		uploadID, err := atb.objLayer.NewMultipartUpload(context.Background(), bucketName,
-			objName, ObjectOptions{})
-		if err != nil {
-			t.Fatalf("mp new error: %v", err)
-		}
-
-		_, err = atb.objLayer.PutObjectPart(context.Background(), bucketName, objName,
-			uploadID, 3, mustGetPutObjReader(t, bytes.NewReader(
-				[]byte("hello")), int64(len("hello")), "", ""), ObjectOptions{})
-		if err != nil {
-			t.Fatalf("mp put error: %v", err)
-		}
-
-	}
-}
-
-func (atb *adminXLTestBed) CleanupHealTestData(t *testing.T) {
-	bucketName := "mybucket"
-	objName := "myobject"
-	for i := 0; i < 10; i++ {
-		atb.objLayer.DeleteObject(context.Background(), bucketName,
-			fmt.Sprintf("%s-%d", objName, i))
-	}
-
-	atb.objLayer.DeleteBucket(context.Background(), bucketName)
-}
-
 // initTestObjLayer - Helper function to initialize an XL-based object
 // layer and set globalObjectAPI.
 func initTestXLObjLayer() (ObjectLayer, []string, error) {
@@ -599,86 +544,6 @@ func TestServiceRestartHandler(t *testing.T) {
 	testServicesCmdHandler(restartCmd, t)
 }
 
-// Test for service set creds management REST API.
-func TestServiceSetCreds(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// Initialize admin peers to make admin RPC calls. Note: In a
-	// single node setup, this degenerates to a simple function
-	// call under the hood.
-	globalMinioAddr = "127.0.0.1:9000"
-
-	credentials := globalServerConfig.GetCredential()
-
-	testCases := []struct {
-		AccessKey          string
-		SecretKey          string
-		EnvKeysSet         bool
-		ExpectedStatusCode int
-	}{
-		// Bad secret key
-		{"minio", "minio", false, http.StatusBadRequest},
-		// Bad  secret key set from the env
-		{"minio", "minio", true, http.StatusMethodNotAllowed},
-		// Good keys set from the env
-		{"minio", "minio123", true, http.StatusMethodNotAllowed},
-		// Successful operation should be the last one to
-		// not change server credentials during tests.
-		{"minio", "minio123", false, http.StatusOK},
-	}
-	for i, testCase := range testCases {
-		// Set or unset environement keys
-		globalIsEnvCreds = testCase.EnvKeysSet
-
-		// Construct setCreds request body
-		body, err := json.Marshal(madmin.SetCredsReq{
-			AccessKey: testCase.AccessKey,
-			SecretKey: testCase.SecretKey})
-		if err != nil {
-			t.Fatalf("JSONify err: %v", err)
-		}
-
-		ebody, err := madmin.EncryptData(credentials.SecretKey, body)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Construct setCreds request
-		req, err := getServiceCmdRequest(setCreds, credentials, ebody)
-		if err != nil {
-			t.Fatalf("Failed to build service status request %v", err)
-		}
-
-		rec := httptest.NewRecorder()
-
-		// Execute request
-		adminTestBed.router.ServeHTTP(rec, req)
-
-		// Check if the http code response is expected
-		if rec.Code != testCase.ExpectedStatusCode {
-			t.Errorf("Test %d: Wrong status code, expected = %d, found = %d", i+1, testCase.ExpectedStatusCode, rec.Code)
-			resp, _ := ioutil.ReadAll(rec.Body)
-			t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
-				http.StatusOK, rec.Code, string(resp))
-		}
-
-		// If we got 200 OK, check if new credentials are really set
-		if rec.Code == http.StatusOK {
-			cred := globalServerConfig.GetCredential()
-			if cred.AccessKey != testCase.AccessKey {
-				t.Errorf("Test %d: Wrong access key, expected = %s, found = %s", i+1, testCase.AccessKey, cred.AccessKey)
-			}
-			if cred.SecretKey != testCase.SecretKey {
-				t.Errorf("Test %d: Wrong secret key, expected = %s, found = %s", i+1, testCase.SecretKey, cred.SecretKey)
-			}
-		}
-	}
-}
-
 // buildAdminRequest - helper function to build an admin API request.
 func buildAdminRequest(queryVal url.Values, method, path string,
 	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {

cmd/admin-router.go

@@ -74,8 +74,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	/// Config operations
 	if enableConfigOps {
-		// Update credentials
-		adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
 		// Get config
 		adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 		// Set config

cmd/api-errors.go

@@ -65,6 +65,7 @@ const (
 	ErrBadDigest
 	ErrEntityTooSmall
 	ErrEntityTooLarge
+	ErrPolicyTooLarge
 	ErrIncompleteBody
 	ErrInternalError
 	ErrInvalidAccessKeyID
@@ -398,6 +399,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your proposed upload exceeds the maximum allowed object size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrPolicyTooLarge: {
+		Code:           "PolicyTooLarge",
+		Description:    "Policy exceeds the maximum allowed document size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrIncompleteBody: {
 		Code:           "IncompleteBody",
 		Description:    "You did not provide the number of bytes specified by the Content-Length HTTP header.",
@@ -1479,7 +1485,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	if err == nil {
 		return ErrNone
 	}
-
 	// Verify if the underlying error is signature mismatch.
 	switch err {
 	case errInvalidArgument:
@@ -1529,6 +1534,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSAuthFailure
 	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
+	case errNetworkConnReset:
+		apiErr = ErrSlowDown
 	}
 
 	// Compression errors

cmd/api-router.go

@@ -46,6 +46,7 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 	var routers []*mux.Router
 	for _, domainName := range globalDomainNames {
 		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
+		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName+":{port:.*}").Subrouter())
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 

cmd/auth-handler.go

@@ -362,7 +362,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 
 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1)
+	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}

cmd/bitrot-streaming.go

@@ -32,33 +32,20 @@ type streamingBitrotWriter struct {
 	h         hash.Hash
 	shardSize int64
 	canClose  chan struct{} // Needed to avoid race explained in Close() call.
-
-	// Following two fields are used only to make sure that Write(p) is called such that
-	// len(p) is always the block size except the last block, i.e prevent programmer errors.
-	currentBlockIdx int
-	verifyTillIdx   int
 }
 
 func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
-	if b.currentBlockIdx < b.verifyTillIdx && int64(len(p)) != b.shardSize {
-		// All blocks except last should be of the length b.shardSize
-		logger.LogIf(context.Background(), errUnexpected)
-		return 0, errUnexpected
-	}
 	if len(p) == 0 {
 		return 0, nil
 	}
 	b.h.Reset()
 	b.h.Write(p)
 	hashBytes := b.h.Sum(nil)
-	n, err := b.iow.Write(hashBytes)
-	if n != len(hashBytes) {
-		logger.LogIf(context.Background(), err)
+	_, err := b.iow.Write(hashBytes)
+	if err != nil {
 		return 0, err
 	}
-	n, err = b.iow.Write(p)
-	b.currentBlockIdx++
-	return n, err
+	return b.iow.Write(p)
 }
 
 func (b *streamingBitrotWriter) Close() error {
@@ -78,16 +65,14 @@ func (b *streamingBitrotWriter) Close() error {
 func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
 	r, w := io.Pipe()
 	h := algo.New()
-	bw := &streamingBitrotWriter{w, h, shardSize, make(chan struct{}), 0, int(length / shardSize)}
+	bw := &streamingBitrotWriter{w, h, shardSize, make(chan struct{})}
 	go func() {
-		bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
-		totalFileSize := bitrotSumsTotalSize + length
-		err := disk.CreateFile(volume, filePath, totalFileSize, r)
-		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("storageDisk", disk.String())
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
+		totalFileSize := int64(-1) // For compressed objects length will be unknown (represented by length=-1)
+		if length != -1 {
+			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
+			totalFileSize = bitrotSumsTotalSize + length
 		}
+		err := disk.CreateFile(volume, filePath, totalFileSize, r)
 		r.CloseWithError(err)
 		close(bw.canClose)
 	}()
@@ -118,7 +103,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	var err error
 	if offset%b.shardSize != 0 {
 		// Offset should always be aligned to b.shardSize
-		logger.LogIf(context.Background(), errUnexpected)
+		// Can never happen unless there are programmer bugs
 		return 0, errUnexpected
 	}
 	if b.rc == nil {
@@ -127,25 +112,20 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
 		b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
 		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("storageDisk", b.disk.String())
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
 			return 0, err
 		}
 	}
 	if offset != b.currOffset {
-		logger.LogIf(context.Background(), errUnexpected)
+		// Can never happen unless there are programmer bugs
 		return 0, errUnexpected
 	}
 	b.h.Reset()
 	_, err = io.ReadFull(b.rc, b.hashBytes)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
 		return 0, err
 	}
 	_, err = io.ReadFull(b.rc, buf)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
 		return 0, err
 	}
 	b.h.Write(buf)

cmd/bitrot-whole.go

@@ -31,19 +31,9 @@ type wholeBitrotWriter struct {
 	filePath  string
 	shardSize int64 // This is the shard size of the erasure logic
 	hash.Hash       // For bitrot hash
-
-	// Following two fields are used only to make sure that Write(p) is called such that
-	// len(p) is always the block size except the last block and prevent programmer errors.
-	currentBlockIdx int
-	lastBlockIdx    int
 }
 
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
-	if b.currentBlockIdx < b.lastBlockIdx && int64(len(p)) != b.shardSize {
-		// All blocks except last should be of the length b.shardSize
-		logger.LogIf(context.Background(), errUnexpected)
-		return 0, errUnexpected
-	}
 	err := b.disk.AppendFile(b.volume, b.filePath, p)
 	if err != nil {
 		logger.LogIf(context.Background(), err)
@@ -54,7 +44,6 @@ func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 		logger.LogIf(context.Background(), err)
 		return 0, err
 	}
-	b.currentBlockIdx++
 	return len(p), nil
 }
 
@@ -63,8 +52,8 @@ func (b *wholeBitrotWriter) Close() error {
 }
 
 // Returns whole-file bitrot writer.
-func newWholeBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
-	return &wholeBitrotWriter{disk, volume, filePath, shardSize, algo.New(), 0, int(length / shardSize)}
+func newWholeBitrotWriter(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
+	return &wholeBitrotWriter{disk, volume, filePath, shardSize, algo.New()}
 }
 
 // Implementation to verify bitrot for the whole file.

cmd/bitrot.go

@@ -120,7 +120,7 @@ func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, alg
 	if algo == HighwayHash256S {
 		return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize)
 	}
-	return newWholeBitrotWriter(disk, volume, filePath, length, algo, shardSize)
+	return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
 }
 
 func newBitrotReader(disk StorageAPI, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {

cmd/bucket-handlers.go

@@ -263,16 +263,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var s3Error APIErrorCode
-	if s3Error = checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, ""); s3Error != ErrNone {
-		// In the event access is denied, a 200 response should still be returned
-		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-		if s3Error != ErrAccessDenied {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
-
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
@@ -319,42 +309,60 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	deleteObject := objectAPI.DeleteObject
+	deleteObjectsFn := objectAPI.DeleteObjects
 	if api.CacheAPI() != nil {
-		deleteObject = api.CacheAPI().DeleteObject
+		deleteObjectsFn = api.CacheAPI().DeleteObjects
 	}
 
-	var dErrs = make([]error, len(deleteObjects.Objects))
+	type delObj struct {
+		origIndex int
+		name      string
+	}
+
+	var objectsToDelete []delObj
+	var dErrs = make([]APIErrorCode, len(deleteObjects.Objects))
+
 	for index, object := range deleteObjects.Objects {
-		// If the request is denied access, each item
-		// should be marked as 'AccessDenied'
-		if s3Error == ErrAccessDenied {
-			dErrs[index] = PrefixAccessDenied{
-				Bucket: bucket,
-				Object: object.ObjectName,
+		if dErrs[index] = checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); dErrs[index] != ErrNone {
+			if dErrs[index] == ErrSignatureDoesNotMatch || dErrs[index] == ErrInvalidAccessKeyID {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(dErrs[index]), r.URL, guessIsBrowserReq(r))
+				return
 			}
 			continue
 		}
-		dErrs[index] = deleteObject(ctx, bucket, object.ObjectName)
+
+		objectsToDelete = append(objectsToDelete, delObj{index, object.ObjectName})
+	}
+
+	toNames := func(input []delObj) (output []string) {
+		output = make([]string, len(input))
+		for i := range input {
+			output[i] = input[i].name
+		}
+		return
+	}
+
+	errs, err := deleteObjectsFn(ctx, bucket, toNames(objectsToDelete))
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	for i, obj := range objectsToDelete {
+		dErrs[obj.origIndex] = toAPIErrorCode(ctx, errs[i])
 	}
 
 	// Collect deleted objects and errors if any.
 	var deletedObjects []ObjectIdentifier
 	var deleteErrors []DeleteError
-	for index, err := range dErrs {
+	for index, errCode := range dErrs {
 		object := deleteObjects.Objects[index]
 		// Success deleted objects are collected separately.
-		if err == nil {
+		if errCode == ErrNone || errCode == ErrNoSuchKey {
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
-		if _, ok := err.(ObjectNotFound); ok {
-			// If the object is not found it should be
-			// accounted as deleted as per S3 spec.
-			deletedObjects = append(deletedObjects, object)
-			continue
-		}
-		apiErr := toAPIError(ctx, err)
+		apiErr := getAPIError(errCode)
 		// Error during delete should be collected separately.
 		deleteErrors = append(deleteErrors, DeleteError{
 			Code:    apiErr.Code,
@@ -615,7 +623,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
+	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -653,7 +661,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				return
 			}
 			info := ObjectInfo{Size: fileSize}
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
+			// do not try to verify encrypted content
+			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize, globalCLIContext.StrictS3Compat)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return

cmd/bucket-handlers_test.go

@@ -762,7 +762,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		apiRouter.ServeHTTP(rec, req)
 		// Assert the response code with the expected status.
 		if rec.Code != testCase.expectedRespStatus {
-			t.Errorf("Case %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+			t.Errorf("Test %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
 		}
 
 		// read the response body.

cmd/bucket-policy-handlers.go

@@ -71,7 +71,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	// Error out if Content-Length is beyond allowed size.
 	if r.ContentLength > maxBucketPolicySize {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyTooLarge), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/common-main.go

@@ -161,6 +161,9 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
 
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
+
+	// Check "compat" flag from command line argument.
+	globalCLIContext.StrictS3Compat = ctx.IsSet("compat") || ctx.GlobalIsSet("compat")
 }
 
 // Parses the given compression exclude list `extensions` or `content-types`.

cmd/config-common.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/minio/minio/cmd/logger"
@@ -73,7 +74,7 @@ func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string,
 }
 
 func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
-	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
+	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)), globalCLIContext.StrictS3Compat)
 	if err != nil {
 		return err
 	}
@@ -105,12 +106,26 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 
 // watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
 func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-	defer cancel()
-	for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
-		for _, event := range watchResp.Events {
-			if event.IsModify() || event.IsCreate() {
-				loadCfgFn(objAPI)
+	for {
+		watchCh := globalEtcdClient.Watch(context.Background(), iamConfigPrefix)
+		select {
+		case <-GlobalServiceDoneCh:
+			return
+		case watchResp, ok := <-watchCh:
+			if !ok {
+				time.Sleep(1 * time.Second)
+				continue
+			}
+			if err := watchResp.Err(); err != nil {
+				logger.LogIf(context.Background(), err)
+				// log and retry.
+				time.Sleep(1 * time.Second)
+				continue
+			}
+			for _, event := range watchResp.Events {
+				if event.IsModify() || event.IsCreate() {
+					loadCfgFn(objAPI)
+				}
 			}
 		}
 	}

cmd/disk-cache-fs.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -399,7 +398,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 	if fsMeta.Meta["etag"] == "" {
-		fsMeta.Meta["etag"] = hex.EncodeToString(data.MD5Current())
+		fsMeta.Meta["etag"] = r.MD5CurrentHexString()
 	}
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.

cmd/disk-cache.go

@@ -62,6 +62,7 @@ type cacheObjects struct {
 	GetObjectInfoFn           func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	PutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn            func(ctx context.Context, bucket, object string) error
+	DeleteObjectsFn           func(ctx context.Context, bucket string, objects []string) ([]error, error)
 	ListObjectsFn             func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2Fn           func(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
 	ListBucketsFn             func(ctx context.Context) (buckets []BucketInfo, err error)
@@ -94,6 +95,7 @@ type CacheObjectLayer interface {
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
+	DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error)
 
 	// Multipart operations.
 	NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
@@ -200,9 +202,12 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		return cacheReader, nil
 	} else if err != nil {
 		if _, ok := err.(ObjectNotFound); ok {
-			cacheReader.Close()
-			// Delete cached entry if backend object was deleted.
-			dcache.Delete(ctx, bucket, object)
+			if cacheErr == nil {
+				cacheReader.Close()
+				// Delete cached entry if backend object
+				// was deleted.
+				dcache.Delete(ctx, bucket, object)
+			}
 		}
 		return nil, err
 	}
@@ -240,7 +245,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
 	teeReader := io.TeeReader(bkReader, pipeWriter)
-	hashReader, herr := hash.NewReader(pipeReader, bkReader.ObjInfo.Size, "", "", bkReader.ObjInfo.Size)
+	hashReader, herr := hash.NewReader(pipeReader, bkReader.ObjInfo.Size, "", "", bkReader.ObjInfo.Size, globalCLIContext.StrictS3Compat)
 	if herr != nil {
 		bkReader.Close()
 		return nil, herr
@@ -311,7 +316,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 	}
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, objInfo.Size, "", "", objInfo.Size)
+	hashReader, err := hash.NewReader(pipeReader, objInfo.Size, "", "", objInfo.Size, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		return err
 	}
@@ -372,7 +377,7 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - list of fsObjects
-func listDirCacheFactory(isLeaf IsLeafFunc, disks []*cacheFSObjects) ListDirFunc {
+func listDirCacheFactory(isLeaf func(string, string) bool, disks []*cacheFSObjects) ListDirFunc {
 	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string) {
 		var entries []string
 		for _, disk := range disks {
@@ -388,6 +393,12 @@ func listDirCacheFactory(isLeaf IsLeafFunc, disks []*cacheFSObjects) ListDirFunc
 				continue
 			}
 
+			for i := range entries {
+				if isLeaf(bucket, entries[i]) {
+					entries[i] = strings.TrimSuffix(entries[i], slashSeparator)
+				}
+			}
+
 			// Filter entries that have the prefix prefixEntry.
 			entries = filterMatchingPrefix(entries, prefixEntry)
 			dirs = append(dirs, entries...)
@@ -396,7 +407,7 @@ func listDirCacheFactory(isLeaf IsLeafFunc, disks []*cacheFSObjects) ListDirFunc
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string) {
 		cacheEntries := listCacheDirs(bucket, prefixDir, prefixEntry)
 		for _, entry := range cacheEntries {
 			// Find elements in entries which are not in mergedEntries
@@ -408,7 +419,7 @@ func listDirCacheFactory(isLeaf IsLeafFunc, disks []*cacheFSObjects) ListDirFunc
 			mergedEntries = append(mergedEntries, entry)
 			sort.Strings(mergedEntries)
 		}
-		return mergedEntries, false
+		return mergedEntries
 	}
 	return listDir
 }
@@ -427,25 +438,16 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 	walkResultCh, endWalkCh := c.listPool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
-		isLeaf := func(bucket, object string) bool {
+
+		listDir := listDirCacheFactory(func(bucket, object string) bool {
 			fs, err := c.cache.getCacheFS(ctx, bucket, object)
 			if err != nil {
 				return false
 			}
 			_, err = fs.getObjectInfo(ctx, bucket, object)
 			return err == nil
-		}
-
-		isLeafDir := func(bucket, object string) bool {
-			fs, err := c.cache.getCacheFS(ctx, bucket, object)
-			if err != nil {
-				return false
-			}
-			return fs.isObjectDir(bucket, object)
-		}
-
-		listDir := listDirCacheFactory(isLeaf, c.cache.cfs)
-		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
+		}, c.cache.cfs)
+		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, endWalkCh)
 	}
 
 	for i := 0; i < maxKeys; {
@@ -455,10 +457,6 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			eof = true
 			break
 		}
-		// For any walk error return right away.
-		if walkResult.err != nil {
-			return result, toObjectErr(walkResult.err, bucket, prefix)
-		}
 
 		entry := walkResult.entry
 		var objInfo ObjectInfo
@@ -633,6 +631,14 @@ func (c cacheObjects) DeleteObject(ctx context.Context, bucket, object string) (
 	return
 }
 
+func (c cacheObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+	errs := make([]error, len(objects))
+	for idx, object := range objects {
+		errs[idx] = c.DeleteObject(ctx, bucket, object)
+	}
+	return errs, nil
+}
+
 // Returns true if object should be excluded from cache
 func (c cacheObjects) isCacheExclude(bucket, object string) bool {
 	for _, pattern := range c.exclude {
@@ -668,13 +674,13 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *P
 	objInfo = ObjectInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
+	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize(), globalCLIContext.StrictS3Compat)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
-	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
+	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize(), globalCLIContext.StrictS3Compat)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -762,13 +768,13 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 	info = PartInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
+	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize(), globalCLIContext.StrictS3Compat)
 	if err != nil {
 		return
 	}
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
-	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
+	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize(), globalCLIContext.StrictS3Compat)
 	if err != nil {
 		return
 	}
@@ -978,6 +984,14 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
 			return newObjectLayerFn().DeleteObject(ctx, bucket, object)
 		},
+		DeleteObjectsFn: func(ctx context.Context, bucket string, objects []string) ([]error, error) {
+			errs := make([]error, len(objects))
+			for idx, object := range objects {
+				errs[idx] = newObjectLayerFn().DeleteObject(ctx, bucket, object)
+			}
+			return errs, nil
+		},
+
 		ListObjectsFn: func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
 			return newObjectLayerFn().ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 		},

cmd/disk-cache_test.go

@@ -194,7 +194,7 @@ func TestDiskCache(t *testing.T) {
 	objInfo.UserDefined = httpMeta
 	var opts ObjectOptions
 	byteReader := bytes.NewReader([]byte(content))
-	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size), globalCLIContext.StrictS3Compat)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -269,7 +269,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 	opts := ObjectOptions{}
 
 	byteReader := bytes.NewReader([]byte(content))
-	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size), globalCLIContext.StrictS3Compat)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/endpoint-ellipses_test.go

@@ -44,7 +44,6 @@ func TestCreateServerEndpoints(t *testing.T) {
 		{":9000", []string{"/export1{1...32}", "/export1{1...32}"}, false},
 		// Same host cannot export same disk on two ports - special case localhost.
 		{":9001", []string{"http://localhost:900{1...2}/export{1...64}"}, false},
-
 		// Valid inputs.
 		{":9000", []string{"/export1"}, true},
 		{":9000", []string{"/export1", "/export2", "/export3", "/export4"}, true},

cmd/endpoint.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -26,7 +27,10 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
+	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/cli"
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/cpu"
@@ -44,6 +48,8 @@ const (
 
 	// URLEndpointType - URL style endpoint type enum.
 	URLEndpointType
+
+	retryInterval = 5 // In Seconds.
 )
 
 // Endpoint - any type of endpoint.
@@ -51,6 +57,7 @@ type Endpoint struct {
 	*url.URL
 	IsLocal  bool
 	SetIndex int
+	HostName string
 }
 
 func (endpoint Endpoint) String() string {
@@ -75,6 +82,19 @@ func (endpoint Endpoint) IsHTTPS() bool {
 	return endpoint.Scheme == "https"
 }
 
+// UpdateIsLocal - resolves the host and updates if it is local or not.
+func (endpoint *Endpoint) UpdateIsLocal() error {
+	if !endpoint.IsLocal {
+		isLocal, err := isLocalHost(endpoint.HostName)
+		if err != nil {
+			return err
+		}
+		endpoint.IsLocal = isLocal
+
+	}
+	return nil
+}
+
 // NewEndpoint - returns new endpoint based on given arguments.
 func NewEndpoint(arg string) (ep Endpoint, e error) {
 	// isEmptyPath - check whether given path is not empty.
@@ -87,6 +107,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	}
 
 	var isLocal bool
+	var host string
 	u, err := url.Parse(arg)
 	if err == nil && u.Host != "" {
 		// URL style of endpoint.
@@ -98,7 +119,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			return ep, fmt.Errorf("invalid URL endpoint format")
 		}
 
-		var host, port string
+		var port string
 		host, port, err = net.SplitHostPort(u.Host)
 		if err != nil {
 			if !strings.Contains(err.Error(), "missing port in address") {
@@ -150,10 +171,6 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			}
 		}
 
-		isLocal, err = isLocalHost(host)
-		if err != nil {
-			return ep, err
-		}
 	} else {
 		// Only check if the arg is an ip address and ask for scheme since its absent.
 		// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
@@ -166,8 +183,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	}
 
 	return Endpoint{
-		URL:     u,
-		IsLocal: isLocal,
+		URL:      u,
+		IsLocal:  isLocal,
+		HostName: host,
 	}, nil
 }
 
@@ -200,6 +218,74 @@ func (endpoints EndpointList) GetString(i int) string {
 	return endpoints[i].String()
 }
 
+// UpdateIsLocal - resolves the host and discovers the local host.
+func (endpoints EndpointList) UpdateIsLocal() error {
+	var epsResolved int
+	var foundLocal bool
+	resolvedList := make([]bool, len(endpoints))
+	// Mark the starting time
+	startTime := time.Now()
+	keepAliveTicker := time.NewTicker(retryInterval * time.Second)
+	defer keepAliveTicker.Stop()
+	for {
+		// Break if the local endpoint is found already. Or all the endpoints are resolved.
+		if foundLocal || (epsResolved == len(endpoints)) {
+			break
+		}
+		// Retry infinitely on Kubernetes and Docker swarm.
+		// This is needed as the remote hosts are sometime
+		// not available immediately.
+		select {
+		case <-globalOSSignalCh:
+			return fmt.Errorf("The endpoint resolution got interrupted")
+		default:
+			for i, resolved := range resolvedList {
+				if resolved {
+					continue
+				}
+
+				// return err if not Docker or Kubernetes
+				// We use IsDocker() method to check for Docker Swarm environment
+				// as there is no reliable way to clearly identify Swarm from
+				// Docker environment.
+				isLocal, err := isLocalHost(endpoints[i].HostName)
+				if err != nil {
+					if !IsDocker() && !IsKubernetes() {
+						return err
+					}
+					// time elapsed
+					timeElapsed := time.Since(startTime)
+					// log error only if more than 1s elapsed
+					if timeElapsed > time.Second {
+						// log the message to console about the host not being
+						// resolveable.
+						reqInfo := (&logger.ReqInfo{}).AppendTags("host", endpoints[i].HostName)
+						reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", ""))
+						ctx := logger.SetReqInfo(context.Background(), reqInfo)
+						logger.LogIf(ctx, err)
+					}
+				} else {
+					resolvedList[i] = true
+					endpoints[i].IsLocal = isLocal
+					epsResolved++
+					if !foundLocal {
+						foundLocal = isLocal
+					}
+				}
+			}
+
+			// Wait for the tick, if the there exist a local endpoint in discovery.
+			// Non docker/kubernetes environment does not need to wait.
+			if !foundLocal && (IsDocker() && IsKubernetes()) {
+				<-keepAliveTicker.C
+			}
+		}
+	}
+
+	return nil
+
+}
+
 // localEndpointsMemUsage - returns ServerMemUsageInfo for only the
 // local endpoints from given list of endpoints
 func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
@@ -302,9 +388,30 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 		uniqueArgs.Add(arg)
 		endpoints = append(endpoints, endpoint)
 	}
+
 	return endpoints, nil
 }
 
+func checkEndpointsSubOptimal(ctx *cli.Context, setupType SetupType, endpoints EndpointList) (err error) {
+	// Validate sub optimal ordering only for distributed setup.
+	if setupType != DistXLSetupType {
+		return nil
+	}
+	var endpointOrder int
+	err = fmt.Errorf("Too many disk args are local, input is in sub-optimal order. Please review input args: %s", ctx.Args())
+	for _, endpoint := range endpoints {
+		if endpoint.IsLocal {
+			endpointOrder++
+		} else {
+			endpointOrder--
+		}
+		if endpointOrder >= 2 {
+			return err
+		}
+	}
+	return nil
+}
+
 // Checks if there are any cross device mounts.
 func checkCrossDeviceMounts(endpoints EndpointList) (err error) {
 	var absPaths []string
@@ -341,6 +448,9 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		if err != nil {
 			return serverAddr, endpoints, setupType, err
 		}
+		if err := endpoint.UpdateIsLocal(); err != nil {
+			return serverAddr, endpoints, setupType, err
+		}
 		if endpoint.Type() != PathEndpointType {
 			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
 		}
@@ -381,6 +491,10 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		return serverAddr, endpoints, setupType, nil
 	}
 
+	if err := endpoints.UpdateIsLocal(); err != nil {
+		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+	}
+
 	// Here all endpoints are URL style.
 	endpointPathSet := set.NewStringSet()
 	localEndpointCount := 0

cmd/endpoint_test.go

@@ -17,13 +17,53 @@
 package cmd
 
 import (
+	"flag"
 	"fmt"
 	"net/url"
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/minio/cli"
 )
 
+func TestSubOptimalEndpointInput(t *testing.T) {
+	args1 := []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}
+	args2 := []string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}
+
+	tests := []struct {
+		setupType SetupType
+		ctx       *cli.Context
+		endpoints EndpointList
+		isErr     bool
+	}{
+		{
+			setupType: DistXLSetupType,
+			ctx:       cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
+			endpoints: mustGetNewEndpointList(args1...),
+			isErr:     false,
+		},
+		{
+			setupType: DistXLSetupType,
+			ctx:       cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
+			endpoints: mustGetNewEndpointList(args2...),
+			isErr:     false,
+		},
+	}
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
+			err := checkEndpointsSubOptimal(test.ctx, test.setupType, test.endpoints)
+			if test.isErr && err == nil {
+				t.Error("expected err but found nil")
+			}
+			if !test.isErr && err != nil {
+				t.Errorf("expected err nil but found an err %s", err)
+			}
+		})
+	}
+}
+
 func TestNewEndpoint(t *testing.T) {
 	u1, _ := url.Parse("http://localhost/path")
 	u2, _ := url.Parse("https://example.org/path")
@@ -49,11 +89,11 @@ func TestNewEndpoint(t *testing.T) {
 		{"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil},
 		{"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
 		{"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
-		{"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
-		{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
-		{"https://example.org/path", Endpoint{URL: u2}, URLEndpointType, nil},
-		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
-		{"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil},
+		{"http://localhost/path", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil},
+		{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil},
+		{"https://example.org/path", Endpoint{URL: u2, IsLocal: false, HostName: "example.org"}, URLEndpointType, nil},
+		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true, HostName: "127.0.0.1"}, URLEndpointType, nil},
+		{"http://192.168.253.200/path", Endpoint{URL: u4, IsLocal: false, HostName: "192.168.253.200"}, URLEndpointType, nil},
 		{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
@@ -71,6 +111,10 @@ func TestNewEndpoint(t *testing.T) {
 
 	for _, testCase := range testCases {
 		endpoint, err := NewEndpoint(testCase.arg)
+		if err == nil {
+			err = endpoint.UpdateIsLocal()
+		}
+
 		if testCase.expectedErr == nil {
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)
@@ -261,46 +305,46 @@ func TestCreateEndpoints(t *testing.T) {
 
 		// DistXL type
 		{"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{
-			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
-			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
-			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
-			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
+			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1], HostName: nonLoopBackIP},
+			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2], HostName: "example.org"},
+			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3], HostName: "example.com"},
 		}, DistXLSetupType, nil},
 
 		{"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{
-			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
-			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
-			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
-			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
+			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1], HostName: nonLoopBackIP},
+			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2], HostName: "example.org"},
+			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3], HostName: "example.com"},
 		}, DistXLSetupType, nil},
 
 		{":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", EndpointList{
-			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
-			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
-			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
-			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
+			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1], HostName: "example.org"},
+			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2], HostName: "example.com"},
+			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3], HostName: "example.net"},
 		}, DistXLSetupType, nil},
 
 		{":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", EndpointList{
-			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
-			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
-			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
-			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
+			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1], HostName: "example.org"},
+			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2], HostName: "example.com"},
+			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3], HostName: "example.net"},
 		}, DistXLSetupType, nil},
 
 		{":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", EndpointList{
-			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
-			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
-			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
-			Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]},
+			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1], HostName: nonLoopBackIP},
+			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2], HostName: nonLoopBackIP},
+			Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3], HostName: nonLoopBackIP},
 		}, DistXLSetupType, nil},
 
 		// DistXL Setup using only local host.
 		{":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", EndpointList{
-			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
-			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
-			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
-			Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]},
+			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0], HostName: "localhost"},
+			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1], HostName: "localhost"},
+			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2], HostName: "127.0.0.1"},
+			Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3], HostName: nonLoopBackIP},
 		}, DistXLSetupType, nil},
 	}
 
@@ -357,6 +401,11 @@ func TestGetLocalPeer(t *testing.T) {
 
 	for i, testCase := range testCases {
 		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		if !endpoints[0].IsLocal {
+			if err := endpoints.UpdateIsLocal(); err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		}
 		remotePeer := GetLocalPeer(endpoints)
 		if remotePeer != testCase.expectedResult {
 			t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, remotePeer)
@@ -384,6 +433,11 @@ func TestGetRemotePeers(t *testing.T) {
 
 	for _, testCase := range testCases {
 		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		if !endpoints[0].IsLocal {
+			if err := endpoints.UpdateIsLocal(); err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		}
 		remotePeers := GetRemotePeers(endpoints)
 		if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
 			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)

cmd/erasure.go

@@ -103,6 +103,9 @@ func (e *Erasure) ShardFileSize(totalLength int64) int64 {
 	if totalLength == 0 {
 		return 0
 	}
+	if totalLength == -1 {
+		return -1
+	}
 	numShards := totalLength / e.blockSize
 	lastBlockSize := totalLength % int64(e.blockSize)
 	lastShardSize := ceilFrac(lastBlockSize, int64(e.dataBlocks))

cmd/fs-v1-helpers.go

@@ -206,6 +206,9 @@ func osErrToFSFileErr(err error) error {
 	if isSysErrPathNotFound(err) {
 		return errFileNotFound
 	}
+	if isSysErrTooManyFiles(err) {
+		return errTooManyOpenFiles
+	}
 	return err
 }
 

cmd/fs-v1-multipart.go

@@ -504,10 +504,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Calculate s3 compatible md5sum for complete multipart.
-	s3MD5, err := getCompleteMultipartMD5(ctx, parts)
-	if err != nil {
-		return oi, err
-	}
+	s3MD5 := getCompleteMultipartMD5(parts)
 
 	partSize := int64(-1) // Used later to ensure that all parts sizes are same.
 

cmd/fs-v1.go

@@ -450,6 +450,11 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		// Return the new object info.
 		return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil
 	}
+
+	if err := checkPutObjectArgs(ctx, dstBucket, dstObject, fs, srcInfo.PutObjReader.Size()); err != nil {
+		return ObjectInfo{}, err
+	}
+
 	objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, ObjectOptions{ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined})
 	if err != nil {
 		return oi, toObjectErr(err, dstBucket, dstObject)
@@ -813,6 +818,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return objInfo, err
 	}
 	defer objectLock.Unlock()
+
 	return fs.putObject(ctx, bucket, object, r, opts)
 }
 
@@ -854,10 +860,6 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 		return fsMeta.ToObjectInfo(bucket, object, fi), nil
 	}
 
-	if err = checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil {
-		return ObjectInfo{}, err
-	}
-
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
 		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
@@ -951,6 +953,16 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	return fsMeta.ToObjectInfo(bucket, object, fi), nil
 }
 
+// DeleteObjects - deletes an object from a bucket, this operation is destructive
+// and there are no rollbacks supported.
+func (fs *FSObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+	errs := make([]error, len(objects))
+	for idx, object := range objects {
+		errs[idx] = fs.DeleteObject(ctx, bucket, object)
+	}
+	return errs, nil
+}
+
 // DeleteObject - deletes an object from a bucket, this operation is destructive
 // and there are no rollbacks supported.
 func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) error {
@@ -1001,17 +1013,17 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry
 // is a leaf or non-leaf entry.
-func (fs *FSObjects) listDirFactory(isLeaf IsLeafFunc) ListDirFunc {
+func (fs *FSObjects) listDirFactory() ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil && err != errFileNotFound {
 			logger.LogIf(context.Background(), err)
 			return
 		}
-		entries, delayIsLeaf = filterListEntries(bucket, prefixDir, entries, prefixEntry, isLeaf)
-		return entries, delayIsLeaf
+		sort.Strings(entries)
+		return filterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.
@@ -1097,22 +1109,8 @@ func (fs *FSObjects) getObjectETag(ctx context.Context, bucket, entry string, lo
 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
 // state for future re-entrant list requests.
 func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	isLeaf := func(bucket, object string) bool {
-		// bucket argument is unused as we don't need to StatFile
-		// to figure if it's a file, just need to check that the
-		// object string does not end with "/".
-		return !hasSuffix(object, slashSeparator)
-	}
-	// Return true if the specified object is an empty directory
-	isLeafDir := func(bucket, object string) bool {
-		if !hasSuffix(object, slashSeparator) {
-			return false
-		}
-		return fs.isObjectDir(bucket, object)
-	}
-	listDir := fs.listDirFactory(isLeaf)
-
-	return listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool, isLeaf, isLeafDir, listDir, fs.getObjectInfo, fs.getObjectInfo)
+	return listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool,
+		fs.listDirFactory(), fs.getObjectInfo, fs.getObjectInfo)
 }
 
 // ReloadFormat - no-op for fs, Valid only for XL.

cmd/gateway-common.go

@@ -43,14 +43,11 @@ var (
 	// ListObjects function alias.
 	ListObjects = listObjects
 
-	// FilterListEntries function alias.
-	FilterListEntries = filterListEntries
+	// FilterMatchingPrefix function alias.
+	FilterMatchingPrefix = filterMatchingPrefix
 
 	// IsStringEqual is string equal.
 	IsStringEqual = isStringEqual
-
-	// GetCompleteMultipartMD5 returns multipart MD5
-	GetCompleteMultipartMD5 = getCompleteMultipartMD5
 )
 
 // StatInfo -  alias for statInfo
@@ -349,6 +346,11 @@ func ErrorRespToObjectError(err error, params ...string) error {
 	return err
 }
 
+// ComputeCompleteMultipartMD5 calculates MD5 ETag for complete multipart responses
+func ComputeCompleteMultipartMD5(parts []CompletePart) string {
+	return getCompleteMultipartMD5(parts)
+}
+
 // parse gateway sse env variable
 func parseGatewaySSE(s string) (gatewaySSE, error) {
 	l := strings.Split(s, ";")

cmd/gateway/azure/gateway-azure.go

@@ -37,6 +37,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/policy"
@@ -89,23 +90,23 @@ ENVIRONMENT VARIABLES:
 
 EXAMPLES:
   1. Start minio gateway server for Azure Blob Storage backend.
-     $ export MINIO_ACCESS_KEY=azureaccountname
-     $ export MINIO_SECRET_KEY=azureaccountkey
-     $ {{.HelpName}}
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
+     {{.Prompt}} {{.HelpName}}
 
   2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
-     $ export MINIO_ACCESS_KEY=azureaccountname
-     $ export MINIO_SECRET_KEY=azureaccountkey
-     $ {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
+     {{.Prompt}} {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint
 
   3. Start minio gateway server for Azure Blob Storage backend with edge caching enabled.
-     $ export MINIO_ACCESS_KEY=azureaccountname
-     $ export MINIO_SECRET_KEY=azureaccountkey
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}}
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
+     {{.Prompt}} {{.HelpName}}
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
@@ -834,7 +835,7 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 	}
 
 	// Save md5sum for future processing on the object.
-	opts.UserDefined["x-amz-meta-md5sum"] = hex.EncodeToString(data.MD5Current())
+	opts.UserDefined["x-amz-meta-md5sum"] = r.MD5CurrentHexString()
 	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
@@ -895,6 +896,14 @@ func (a *azureObjects) DeleteObject(ctx context.Context, bucket, object string)
 	return nil
 }
 
+func (a *azureObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+	errs := make([]error, len(objects))
+	for idx, object := range objects {
+		errs[idx] = a.DeleteObject(ctx, bucket, object)
+	}
+	return errs, nil
+}
+
 // ListMultipartUploads - It's decided not to support List Multipart Uploads, hence returning empty result.
 func (a *azureObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result minio.ListMultipartsInfo, err error) {
 	// It's decided not to support List Multipart Uploads, hence returning empty result.
@@ -1202,19 +1211,18 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	if len(metadata.Metadata) > 0 {
-		objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, metadata.Metadata)
-		if err != nil {
-			return objInfo, azureToObjectError(err, bucket, object)
-		}
-		err = objBlob.SetProperties(nil)
-		if err != nil {
-			return objInfo, azureToObjectError(err, bucket, object)
-		}
-		err = objBlob.SetMetadata(nil)
-		if err != nil {
-			return objInfo, azureToObjectError(err, bucket, object)
-		}
+	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, metadata.Metadata)
+	if err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
+	objBlob.Metadata["md5sum"] = cmd.ComputeCompleteMultipartMD5(uploadedParts)
+	err = objBlob.SetProperties(nil)
+	if err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
+	err = objBlob.SetMetadata(nil)
+	if err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
 	}
 	var partNumberMarker int
 	for {

cmd/gateway/b2/gateway-b2.go

@@ -77,18 +77,18 @@ ENVIRONMENT VARIABLES:
 
 EXAMPLES:
   1. Start minio gateway server for B2 backend.
-     $ export MINIO_ACCESS_KEY=accountID
-     $ export MINIO_SECRET_KEY=applicationKey
-     $ {{.HelpName}}
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accountID
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey
+     {{.Prompt}} {{.HelpName}}
 
   2. Start minio gateway server for B2 backend with edge caching enabled.
-     $ export MINIO_ACCESS_KEY=accountID
-     $ export MINIO_SECRET_KEY=applicationKey
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}}
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accountID
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
+     {{.Prompt}} {{.HelpName}}
 `
 	minio.RegisterGatewayCommand(cli.Command{
 		Name:               b2Backend,
@@ -595,6 +595,14 @@ func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object stri
 	return b2ToObjectError(err, bucket, object)
 }
 
+func (l *b2Objects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+	errs := make([]error, len(objects))
+	for idx, object := range objects {
+		errs[idx] = l.DeleteObject(ctx, bucket, object)
+	}
+	return errs, nil
+}
+
 // ListMultipartUploads lists all multipart uploads.
 func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string,
 	delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {

cmd/gateway/gcs/gateway-gcs.go

@@ -19,6 +19,7 @@ package gcs
 import (
 	"context"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -128,21 +129,21 @@ ENVIRONMENT VARIABLES:
 
 EXAMPLES:
   1. Start minio gateway server for GCS backend.
-     $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
+     {{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json
      (Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials)
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ {{.HelpName}} mygcsprojectid
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.HelpName}} mygcsprojectid
 
   2. Start minio gateway server for GCS backend with edge caching enabled.
-     $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}} mygcsprojectid
+     {{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
+     {{.Prompt}} {{.HelpName}} mygcsprojectid
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
@@ -830,12 +831,17 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
 	if attrs.ContentLanguage != "" {
 		metadata["Content-Language"] = attrs.ContentLanguage
 	}
+
+	etag := hex.EncodeToString(attrs.MD5)
+	if etag == "" {
+		etag = minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C))
+	}
 	return minio.ObjectInfo{
 		Name:            attrs.Name,
 		Bucket:          attrs.Bucket,
 		ModTime:         attrs.Updated,
 		Size:            attrs.Size,
-		ETag:            minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C)),
+		ETag:            etag,
 		UserDefined:     metadata,
 		ContentType:     attrs.ContentType,
 		ContentEncoding: attrs.ContentEncoding,
@@ -915,15 +921,12 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
 	}
 
 	// Close the object writer upon success.
-	w.Close()
-
-	attrs, err := object.Attrs(ctx)
-	if err != nil {
+	if err := w.Close(); err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
 	}
 
-	return fromGCSAttrsToObjectInfo(attrs), nil
+	return fromGCSAttrsToObjectInfo(w.Attrs()), nil
 }
 
 // CopyObject - Copies a blob from source container to destination container.
@@ -958,6 +961,14 @@ func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object str
 	return nil
 }
 
+func (l *gcsGateway) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+	errs := make([]error, len(objects))
+	for idx, object := range objects {
+		errs[idx] = l.DeleteObject(ctx, bucket, object)
+	}
+	return errs, nil
+}
+
 // NewMultipartUpload - upload object in multiple parts
 func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, o minio.ObjectOptions) (uploadID string, err error) {
 	// generate new uploadid
@@ -1093,7 +1104,10 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin
 		return minio.PartInfo{}, gcsToObjectError(err, bucket, key)
 	}
 	// Make sure to close the object writer upon success.
-	w.Close()
+	if err := w.Close(); err != nil {
+		logger.LogIf(ctx, err)
+		return minio.PartInfo{}, gcsToObjectError(err, bucket, key)
+	}
 	return minio.PartInfo{
 		PartNumber:   partNumber,
 		ETag:         etag,

cmd/gateway/hdfs/gateway-hdfs.go

@@ -29,8 +29,8 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/colinmarc/hdfs/v2"
 	"github.com/minio/cli"
+	"github.com/minio/hdfs/v3"
 	"github.com/minio/minio-go/pkg/s3utils"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
@@ -76,18 +76,18 @@ ENVIRONMENT VARIABLES:
 
 EXAMPLES:
   1. Start minio gateway server for HDFS backend.
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ {{.HelpName}} hdfs://namenode:8200
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.HelpName}} hdfs://namenode:8200
 
   2. Start minio gateway server for HDFS with edge caching enabled.
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}} hdfs://namenode:8200
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
+     {{.Prompt}} {{.HelpName}} hdfs://namenode:8200
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
@@ -287,24 +287,9 @@ func (n *hdfsObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketIn
 	return buckets, nil
 }
 
-func (n *hdfsObjects) isObjectDir(bucket, object string) bool {
-	f, err := n.clnt.Open(minio.PathJoin(hdfsSeparator, bucket, object))
-	if err != nil {
-		return false
-	}
-	defer f.Close()
-
-	entries, err := f.Readdir(1)
-	if err != nil {
-		return false
-	}
-
-	return len(entries) == 0
-}
-
-func (n *hdfsObjects) listDirFactory(isLeaf minio.IsLeafFunc) minio.ListDirFunc {
+func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
 		f, err := n.clnt.Open(minio.PathJoin(hdfsSeparator, bucket, prefixDir))
 		if err != nil {
 			if os.IsNotExist(err) {
@@ -327,8 +312,7 @@ func (n *hdfsObjects) listDirFactory(isLeaf minio.IsLeafFunc) minio.ListDirFunc
 			}
 		}
 		fis = nil
-		entries, delayIsLeaf = minio.FilterListEntries(bucket, prefixDir, entries, prefixEntry, isLeaf)
-		return entries, delayIsLeaf
+		return minio.FilterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.
@@ -337,27 +321,11 @@ func (n *hdfsObjects) listDirFactory(isLeaf minio.IsLeafFunc) minio.ListDirFunc
 
 // ListObjects lists all blobs in HDFS bucket filtered by prefix.
 func (n *hdfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
-	isLeaf := func(bucket, object string) bool {
-		// bucket argument is unused as we don't need to StatFile
-		// to figure if it's a file, just need to check that the
-		// object string does not end with "/".
-		return !strings.HasSuffix(object, hdfsSeparator)
-	}
-
-	// Return true if the specified object is an empty directory
-	isLeafDir := func(bucket, object string) bool {
-		if !strings.HasSuffix(object, hdfsSeparator) {
-			return false
-		}
-		return n.isObjectDir(bucket, object)
-	}
-	listDir := n.listDirFactory(isLeaf)
-
 	getObjectInfo := func(ctx context.Context, bucket, entry string) (minio.ObjectInfo, error) {
 		return n.GetObjectInfo(ctx, bucket, entry, minio.ObjectOptions{})
 	}
 
-	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, isLeaf, isLeafDir, listDir, getObjectInfo, getObjectInfo)
+	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, n.listDirFactory(), getObjectInfo, getObjectInfo)
 }
 
 // Check if the given error corresponds to ENOTEMPTY for unix
@@ -432,6 +400,14 @@ func (n *hdfsObjects) DeleteObject(ctx context.Context, bucket, object string) e
 	return hdfsToObjectErr(ctx, n.deleteObject(minio.PathJoin(hdfsSeparator, bucket), minio.PathJoin(hdfsSeparator, bucket, object)), bucket, object)
 }
 
+func (n *hdfsObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+	errs := make([]error, len(objects))
+	for idx, object := range objects {
+		errs[idx] = n.DeleteObject(ctx, bucket, object)
+	}
+	return errs, nil
+}
+
 func (n *hdfsObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
 	objInfo, err := n.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
@@ -477,13 +453,15 @@ func (n *hdfsObjects) GetObject(ctx context.Context, bucket, key string, startOf
 	if err != nil {
 		return hdfsToObjectErr(ctx, err, bucket, key)
 	}
-	if _, err = rd.Seek(startOffset, io.SeekStart); err != nil {
-		return hdfsToObjectErr(ctx, err, bucket, key)
+	defer rd.Close()
+	_, err = io.Copy(writer, io.NewSectionReader(rd, startOffset, length))
+	if err == io.ErrClosedPipe {
+		// hdfs library doesn't send EOF correctly, so io.Copy attempts
+		// to write which returns io.ErrClosedPipe - just ignore
+		// this for now.
+		err = nil
 	}
-	if _, err = io.Copy(writer, io.LimitReader(rd, length)); err != nil {
-		return hdfsToObjectErr(ctx, err, bucket, key)
-	}
-	return nil
+	return hdfsToObjectErr(ctx, err, bucket, key)
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo.
@@ -528,17 +506,19 @@ func (n *hdfsObjects) PutObject(ctx context.Context, bucket string, object strin
 			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
 		}
 		defer n.deleteObject(minio.PathJoin(hdfsSeparator, minioMetaTmpBucket), tmpname)
-		defer w.Close()
 		if _, err = io.Copy(w, r); err != nil {
+			w.Close()
 			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
 		}
 		dir := path.Dir(name)
 		if dir != "" {
 			if err = n.clnt.MkdirAll(dir, os.FileMode(0755)); err != nil {
+				w.Close()
 				n.deleteObject(minio.PathJoin(hdfsSeparator, bucket), dir)
 				return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
 			}
 		}
+		w.Close()
 		if err = n.clnt.Rename(tmpname, name); err != nil {
 			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
 		}
@@ -675,10 +655,7 @@ func (n *hdfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, objec
 	}
 
 	// Calculate s3 compatible md5sum for complete multipart.
-	s3MD5, err := minio.GetCompleteMultipartMD5(ctx, parts)
-	if err != nil {
-		return objInfo, err
-	}
+	s3MD5 := minio.ComputeCompleteMultipartMD5(parts)
 
 	return minio.ObjectInfo{
 		Bucket:  bucket,

cmd/gateway/nas/gateway-nas.go

@@ -60,18 +60,18 @@ ENVIRONMENT VARIABLES:
 
 EXAMPLES:
   1. Start minio gateway server for NAS backend.
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ {{.HelpName}} /shared/nasvol
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.HelpName}} /shared/nasvol
 
   2. Start minio gateway server for NAS with edge caching enabled.
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}} /shared/nasvol
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
+     {{.Prompt}} {{.HelpName}} /shared/nasvol
 `
 
 	minio.RegisterGatewayCommand(cli.Command{

cmd/gateway/oss/gateway-oss.go

@@ -79,23 +79,23 @@ ENVIRONMENT VARIABLES:
 
 EXAMPLES:
   1. Start minio gateway server for Aliyun OSS backend.
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ {{.HelpName}}
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.HelpName}}
 
   2. Start minio gateway server for Aliyun OSS backend on custom endpoint.
-     $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
-     $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-     $ {{.HelpName}} https://oss.example.com
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+     {{.Prompt}} {{.HelpName}} https://oss.example.com
 
   3. Start minio gateway server for Aliyun OSS backend with edge caching enabled.
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}}
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
+     {{.Prompt}} {{.HelpName}}
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
@@ -710,6 +710,14 @@ func (l *ossObjects) DeleteObject(ctx context.Context, bucket, object string) er
 	return nil
 }
 
+func (l *ossObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+	errs := make([]error, len(objects))
+	for idx, object := range objects {
+		errs[idx] = l.DeleteObject(ctx, bucket, object)
+	}
+	return errs, nil
+}
+
 // fromOSSClientListMultipartsInfo converts oss ListMultipartUploadResult to ListMultipartsInfo
 func fromOSSClientListMultipartsInfo(lmur oss.ListMultipartUploadResult) minio.ListMultipartsInfo {
 	uploads := make([]minio.MultipartInfo, len(lmur.Uploads))

cmd/gateway/s3/gateway-s3-metadata.go

@@ -241,7 +241,7 @@ func getGWMetadata(ctx context.Context, bucket, prefix string, gwMeta gwMetaV1)
 		logger.LogIf(ctx, err)
 		return nil, err
 	}
-	hashReader, err := hash.NewReader(bytes.NewReader(metadataBytes), int64(len(metadataBytes)), "", "", int64(len(metadataBytes)))
+	hashReader, err := hash.NewReader(bytes.NewReader(metadataBytes), int64(len(metadataBytes)), "", "", int64(len(metadataBytes)), false)
 	if err != nil {
 		return nil, err
 	}

cmd/gateway/s3/gateway-s3.go

@@ -77,38 +77,38 @@ ENVIRONMENT VARIABLES:
 
 EXAMPLES:
   1. Start minio gateway server for AWS S3 backend.
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ {{.HelpName}}
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.HelpName}}
 
   2. Start minio gateway server for S3 backend on custom endpoint.
-     $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
-     $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-     $ {{.HelpName}} https://play.min.io:9000
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+     {{.Prompt}} {{.HelpName}} https://play.min.io:9000
 
   3. Start minio gateway server for AWS S3 backend logging all requests to http endpoint.
-     $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
-     $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-     $ export MINIO_LOGGER_HTTP_ENDPOINT="http://localhost:8000/"
-     $ {{.HelpName}} https://play.min.io:9000
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}"http://localhost:8000/"
+     {{.Prompt}} {{.HelpName}} https://play.min.io:9000
 
   4. Start minio gateway server for AWS S3 backend with edge caching enabled.
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}}
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}"bucket1/*;*.png"
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}40
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}80
+     {{.Prompt}} {{.HelpName}}
 
   4. Start minio gateway server for AWS S3 backend using AWS environment variables.
      NOTE: The access and secret key in this case will authenticate with MinIO instead
      of AWS and AWS envs will be used to authenticate to AWS S3.
-     $ export AWS_ACCESS_KEY_ID=aws_access_key
-     $ export AWS_SECRET_ACCESS_KEY=aws_secret_key
-     $ export MINIO_ACCESS_KEY=accesskey
-     $ export MINIO_SECRET_KEY=secretkey
-     $ {{.HelpName}}
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}aws_access_key
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}aws_secret_key
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.HelpName}}
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
@@ -500,6 +500,14 @@ func (l *s3Objects) DeleteObject(ctx context.Context, bucket string, object stri
 	return nil
 }
 
+func (l *s3Objects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+	errs := make([]error, len(objects))
+	for idx, object := range objects {
+		errs[idx] = l.DeleteObject(ctx, bucket, object)
+	}
+	return errs, nil
+}
+
 // ListMultipartUploads lists all multipart uploads.
 func (l *s3Objects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, e error) {
 	result, err := l.Client.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)

cmd/generic-handlers.go

@@ -31,9 +31,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/handlers"
-	"github.com/minio/minio/pkg/sys"
 	"github.com/rs/cors"
-	"golang.org/x/time/rate"
 )
 
 // HandlerFunc - useful to chain different middleware http.Handler
@@ -737,46 +735,6 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 	return bucketForwardingHandler{fwd, h}
 }
 
-// setRateLimitHandler middleware limits the throughput to h using a
-// rate.Limiter token bucket configured with maxOpenFileLimit and
-// burst set to 1. The request will idle for up to 1*time.Second.
-// If the limiter detects the deadline will be exceeded, the request is
-// canceled immediately.
-func setRateLimitHandler(h http.Handler) http.Handler {
-	_, maxLimit, err := sys.GetMaxOpenFileLimit()
-	logger.FatalIf(err, "Unable to get maximum open file limit")
-	// Burst value is set to 1 to allow only maxOpenFileLimit
-	// requests to happen at once.
-	l := rate.NewLimiter(rate.Limit(maxLimit), 1)
-	return rateLimit{l, h}
-}
-
-type rateLimit struct {
-	*rate.Limiter
-	handler http.Handler
-}
-
-func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// create a new context from the request with the wait timeout
-	ctx, cancel := context.WithTimeout(r.Context(), 1*time.Second)
-	defer cancel() // always cancel the context!
-
-	// Wait errors out if the request cannot be processed within
-	// the deadline. time/rate tries to reserve a slot if possible
-	// with in the given duration if it's not possible then Wait(ctx)
-	// returns an error and we cancel the request with ErrSlowDown
-	// error message to the client. This context wait also ensures
-	// requests doomed to fail are terminated early, preventing a
-	// potential pileup on the server.
-	if err := l.Wait(ctx); err != nil {
-		// Send an S3 compatible error, SlowDown.
-		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrSlowDown), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	l.handler.ServeHTTP(w, r)
-}
-
 // customHeaderHandler sets x-amz-request-id, x-minio-deployment-id header.
 // Previously, this value was set right before a response was sent to
 // the client. So, logger and Error response XML were not using this

cmd/globals.go

@@ -92,9 +92,10 @@ const (
 )
 
 var globalCLIContext = struct {
-	JSON, Quiet bool
-	Anonymous   bool
-	Addr        string
+	JSON, Quiet    bool
+	Anonymous      bool
+	Addr           string
+	StrictS3Compat bool
 }{}
 
 var (
@@ -208,13 +209,6 @@ var (
 	// Max allowed disk cache percentage
 	globalCacheMaxUse = 80
 
-	// RPC V1 - Initial version
-	// RPC V2 - format.json XL version changed to 2
-	// RPC V3 - format.json XL version changed to 3
-	// RPC V4 - ReadFile() arguments signature changed
-	// Current RPC version
-	globalRPCAPIVersion = RPCVersion{4, 0, 0}
-
 	// Allocated etcd endpoint for config and bucket DNS.
 	globalEtcdClient *etcd.Client
 

cmd/http/conn_bug_21133.go

@@ -1,48 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package http
-
-import (
-	"net"
-	"sync/atomic"
-	"time"
-)
-
-// QuirkConn - similar to golang net.Conn struct, but contains a workaround of the
-// following the go bug reported here https://github.com/golang/go/issues/21133.
-// Once the bug will be fixed, we can remove this structure and replaces it with
-// the standard net.Conn
-type QuirkConn struct {
-	net.Conn
-	hadReadDeadlineInPast int32 // atomic
-}
-
-// SetReadDeadline - implements a workaround of SetReadDeadline go bug
-func (q *QuirkConn) SetReadDeadline(t time.Time) error {
-	inPast := int32(0)
-	if t.Before(time.Now()) {
-		inPast = 1
-	}
-	atomic.StoreInt32(&q.hadReadDeadlineInPast, inPast)
-	return q.Conn.SetReadDeadline(t)
-}
-
-// canSetReadDeadline - returns if it is safe to set a new
-// read deadline without triggering golang/go#21133 issue.
-func (q *QuirkConn) canSetReadDeadline() bool {
-	return atomic.LoadInt32(&q.hadReadDeadlineInPast) != 1
-}

cmd/http/deadlineconn.go

@@ -23,7 +23,7 @@ import (
 
 // DeadlineConn - is a generic stream-oriented network connection supporting buffered reader and read/write timeout.
 type DeadlineConn struct {
-	QuirkConn
+	net.Conn
 	readTimeout            time.Duration // sets the read timeout in the connection.
 	writeTimeout           time.Duration // sets the write timeout in the connection.
 	updateBytesReadFunc    func(int)     // function to be called to update bytes read.
@@ -32,7 +32,7 @@ type DeadlineConn struct {
 
 // Sets read timeout
 func (c *DeadlineConn) setReadTimeout() {
-	if c.readTimeout != 0 && c.canSetReadDeadline() {
+	if c.readTimeout != 0 {
 		c.SetReadDeadline(time.Now().UTC().Add(c.readTimeout))
 	}
 }
@@ -68,7 +68,7 @@ func (c *DeadlineConn) Write(b []byte) (n int, err error) {
 // newDeadlineConn - creates a new connection object wrapping net.Conn with deadlines.
 func newDeadlineConn(c net.Conn, readTimeout, writeTimeout time.Duration, updateBytesReadFunc, updateBytesWrittenFunc func(int)) *DeadlineConn {
 	return &DeadlineConn{
-		QuirkConn:              QuirkConn{Conn: c},
+		Conn:                   c,
 		readTimeout:            readTimeout,
 		writeTimeout:           writeTimeout,
 		updateBytesReadFunc:    updateBytesReadFunc,

cmd/http/timeoutconn.go

@@ -1,62 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package http
-
-import (
-	"net"
-	"time"
-)
-
-// TimeoutConn - is wrapped net.Conn with read/write timeouts.
-type TimeoutConn struct {
-	QuirkConn
-	readTimeout  time.Duration
-	writeTimeout time.Duration
-}
-
-func (c *TimeoutConn) setReadTimeout() {
-	if c.readTimeout != 0 && c.canSetReadDeadline() {
-		c.SetReadDeadline(time.Now().UTC().Add(c.readTimeout))
-	}
-}
-
-func (c *TimeoutConn) setWriteTimeout() {
-	if c.writeTimeout != 0 {
-		c.SetWriteDeadline(time.Now().UTC().Add(c.writeTimeout))
-	}
-}
-
-// Read - reads data from the connection with timeout.
-func (c *TimeoutConn) Read(b []byte) (n int, err error) {
-	c.setReadTimeout()
-	return c.Conn.Read(b)
-}
-
-// Write - writes data to the connection with timeout.
-func (c *TimeoutConn) Write(b []byte) (n int, err error) {
-	c.setWriteTimeout()
-	return c.Conn.Write(b)
-}
-
-// NewTimeoutConn - creates a new timeout connection.
-func NewTimeoutConn(c net.Conn, readTimeout, writeTimeout time.Duration) *TimeoutConn {
-	return &TimeoutConn{
-		QuirkConn:    QuirkConn{Conn: c},
-		readTimeout:  readTimeout,
-		writeTimeout: writeTimeout,
-	}
-}

cmd/iam.go

@@ -62,6 +62,9 @@ type IAMSys struct {
 
 // Load - loads iam subsystem
 func (sys *IAMSys) Load(objAPI ObjectLayer) error {
+	if globalEtcdClient != nil {
+		return sys.refreshEtcd()
+	}
 	return sys.refresh(objAPI)
 }
 
@@ -71,21 +74,52 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	defer func() {
-		// Refresh IAMSys in background.
-		go func() {
-			ticker := time.NewTicker(globalRefreshIAMInterval)
-			defer ticker.Stop()
-			for {
-				select {
-				case <-GlobalServiceDoneCh:
-					return
-				case <-ticker.C:
-					sys.refresh(objAPI)
+	if globalEtcdClient != nil {
+		defer func() {
+			go func() {
+				// Refresh IAMSys with etcd watch.
+				for {
+					watchCh := globalEtcdClient.Watch(context.Background(), iamConfigPrefix)
+					select {
+					case <-GlobalServiceDoneCh:
+						return
+					case watchResp, ok := <-watchCh:
+						if !ok {
+							time.Sleep(1 * time.Second)
+							continue
+						}
+						if err := watchResp.Err(); err != nil {
+							logger.LogIf(context.Background(), err)
+							// log and retry.
+							time.Sleep(1 * time.Second)
+							continue
+						}
+						for _, event := range watchResp.Events {
+							if event.IsModify() || event.IsCreate() || event.Type == etcd.EventTypeDelete {
+								sys.refreshEtcd()
+							}
+						}
+					}
 				}
-			}
+			}()
 		}()
-	}()
+	} else {
+		defer func() {
+			// Refresh IAMSys in background.
+			go func() {
+				ticker := time.NewTicker(globalRefreshIAMInterval)
+				defer ticker.Stop()
+				for {
+					select {
+					case <-GlobalServiceDoneCh:
+						return
+					case <-ticker.C:
+						sys.refresh(objAPI)
+					}
+				}
+			}()
+		}()
+	}
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
@@ -95,6 +129,9 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 	//  - Read quorum is lost just after the initialization
 	//    of the object layer.
 	for range newRetryTimerSimple(doneCh) {
+		if globalEtcdClient != nil {
+			return sys.refreshEtcd()
+		}
 		// Load IAMSys once during boot.
 		if err := sys.refresh(objAPI); err != nil {
 			if err == errDiskNotFound ||
@@ -697,35 +734,51 @@ func setDefaultCannedPolicies(policies map[string]iampolicy.Policy) {
 	}
 }
 
+func (sys *IAMSys) refreshEtcd() error {
+	iamUsersMap := make(map[string]auth.Credentials)
+	iamPolicyMap := make(map[string]string)
+	iamCannedPolicyMap := make(map[string]iampolicy.Policy)
+
+	if err := reloadEtcdPolicies(iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
+		return err
+	}
+	if err := reloadEtcdUsers(iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
+		return err
+	}
+	if err := reloadEtcdUsers(iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
+		return err
+	}
+
+	// Sets default canned policies, if none are set.
+	setDefaultCannedPolicies(iamCannedPolicyMap)
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.iamUsersMap = iamUsersMap
+	sys.iamPolicyMap = iamPolicyMap
+	sys.iamCannedPolicyMap = iamCannedPolicyMap
+
+	return nil
+}
+
 // Refresh IAMSys.
 func (sys *IAMSys) refresh(objAPI ObjectLayer) error {
 	iamUsersMap := make(map[string]auth.Credentials)
 	iamPolicyMap := make(map[string]string)
 	iamCannedPolicyMap := make(map[string]iampolicy.Policy)
 
-	if globalEtcdClient != nil {
-		if err := reloadEtcdPolicies(iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
-			return err
-		}
-		if err := reloadEtcdUsers(iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
-			return err
-		}
-		if err := reloadEtcdUsers(iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
-			return err
-		}
-	} else {
-		if err := reloadPolicies(objAPI, iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
-			return err
-		}
-		if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
-			return err
-		}
-		if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
-			return err
-		}
+	if err := reloadPolicies(objAPI, iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
+		return err
+	}
+	if err := reloadUsers(objAPI, iamConfigUsersPrefix, iamUsersMap, iamPolicyMap); err != nil {
+		return err
+	}
+	if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
+		return err
 	}
 
-	// Sets default canned policies, if none set.
+	// Sets default canned policies, if none are set.
 	setDefaultCannedPolicies(iamCannedPolicyMap)
 
 	sys.Lock()

cmd/lock-rest-client.go

@@ -0,0 +1,223 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/gob"
+	"errors"
+	"io"
+	"sync"
+	"time"
+
+	"net/url"
+
+	"github.com/minio/dsync"
+	"github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/rest"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+// lockRESTClient is authenticable lock REST client
+type lockRESTClient struct {
+	lockSync   sync.RWMutex
+	host       *xnet.Host
+	restClient *rest.Client
+	serverURL  *url.URL
+	connected  bool
+	timer      *time.Timer
+}
+
+// ServerAddr - dsync.NetLocker interface compatible method.
+func (client *lockRESTClient) ServerAddr() string {
+	return client.serverURL.Host
+}
+
+// ServiceEndpoint - dsync.NetLocker interface compatible method.
+func (client *lockRESTClient) ServiceEndpoint() string {
+	return client.serverURL.Path
+}
+
+// check if the host is up or if it is fine
+// to make a call to the lock rest server.
+func (client *lockRESTClient) isHostUp() bool {
+	client.lockSync.Lock()
+	defer client.lockSync.Unlock()
+
+	if client.connected {
+		return true
+	}
+	select {
+	case <-client.timer.C:
+		client.connected = true
+		client.timer = nil
+		return true
+	default:
+	}
+	return false
+}
+
+// Mark the host as down if there is a Network error.
+func (client *lockRESTClient) markHostDown() {
+	client.lockSync.Lock()
+	defer client.lockSync.Unlock()
+
+	if !client.connected {
+		return
+	}
+	client.connected = false
+	client.timer = time.NewTimer(defaultRetryUnit * 5)
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
+// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *lockRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+
+	if !client.isHostUp() {
+		return nil, errors.New("Lock rest server node is down")
+	}
+
+	if values == nil {
+		values = make(url.Values)
+	}
+
+	respBody, err = client.restClient.Call(method, values, body, length)
+
+	if err == nil {
+		return respBody, nil
+	}
+
+	if isNetworkError(err) {
+		client.markHostDown()
+	}
+
+	return nil, err
+}
+
+// Stringer provides a canonicalized representation of node.
+func (client *lockRESTClient) String() string {
+	return client.host.String()
+}
+
+// IsOnline - returns whether REST client failed to connect or not.
+func (client *lockRESTClient) IsOnline() bool {
+	return client.connected
+}
+
+// Close - marks the client as closed.
+func (client *lockRESTClient) Close() error {
+	client.connected = false
+	client.restClient.Close()
+	return nil
+}
+
+// restCall makes a call to the lock REST server.
+func (client *lockRESTClient) restCall(call string, args dsync.LockArgs) (reply bool, err error) {
+
+	reader := bytes.NewBuffer(make([]byte, 0, 2048))
+	err = gob.NewEncoder(reader).Encode(args)
+	if err != nil {
+		return false, err
+	}
+	respBody, err := client.call(call, nil, reader, -1)
+	if err != nil {
+		return false, err
+	}
+
+	var resp lockResponse
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&resp)
+
+	if err != nil || !resp.Success {
+		reqInfo := &logger.ReqInfo{}
+		reqInfo.AppendTags("resource", args.Resource)
+		reqInfo.AppendTags("serveraddress", args.ServerAddr)
+		reqInfo.AppendTags("serviceendpoint", args.ServiceEndpoint)
+		reqInfo.AppendTags("source", args.Source)
+		reqInfo.AppendTags("uid", args.UID)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+	}
+	return resp.Success, err
+}
+
+// RLock calls read lock REST API.
+func (client *lockRESTClient) RLock(args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(lockRESTMethodRLock, args)
+}
+
+// Lock calls lock REST API.
+func (client *lockRESTClient) Lock(args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(lockRESTMethodLock, args)
+}
+
+// RUnlock calls read unlock REST API.
+func (client *lockRESTClient) RUnlock(args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(lockRESTMethodRUnlock, args)
+}
+
+// Unlock calls write unlock RPC.
+func (client *lockRESTClient) Unlock(args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(lockRESTMethodUnlock, args)
+}
+
+// ForceUnlock calls force unlock RPC.
+func (client *lockRESTClient) ForceUnlock(args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(lockRESTMethodForceUnlock, args)
+}
+
+// Expired calls expired RPC.
+func (client *lockRESTClient) Expired(args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(lockRESTMethodExpired, args)
+}
+
+// Returns a lock rest client.
+func newlockRESTClient(peer *xnet.Host) *lockRESTClient {
+
+	scheme := "http"
+	if globalIsSSL {
+		scheme = "https"
+	}
+
+	serverURL := &url.URL{
+		Scheme: scheme,
+		Host:   peer.String(),
+		Path:   lockRESTPath,
+	}
+
+	var tlsConfig *tls.Config
+	if globalIsSSL {
+		tlsConfig = &tls.Config{
+			ServerName: peer.Name,
+			RootCAs:    globalRootCAs,
+			NextProtos: []string{"http/1.1"}, // Force http1.1
+		}
+	}
+
+	restClient, err := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
+
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: false, timer: time.NewTimer(defaultRetryUnit * 5)}
+	}
+
+	return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: true}
+}

cmd/lock-rest-client_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,14 +24,14 @@ import (
 )
 
 // Tests lock rpc client.
-func TestLockRPCClient(t *testing.T) {
+func TestLockRESTlient(t *testing.T) {
 	host, err := xnet.ParseHost("localhost:9000")
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}
-	lkClient, err := NewLockRPCClient(host)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
+	lkClient := newlockRESTClient(host)
+	if lkClient.connected == false {
+		t.Fatalf("unexpected error. connection failed")
 	}
 
 	// Attempt all calls.

cmd/lock-rest-server-common.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018, 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,11 +17,22 @@
 package cmd
 
 import (
-	"context"
-	"errors"
+	"path"
 	"time"
+)
 
-	"github.com/minio/minio/cmd/logger"
+const lockRESTVersion = "v1"
+const lockRESTPath = minioReservedBucketPath + "/lock/" + lockRESTVersion
+
+var lockServicePath = path.Join(minioReservedBucketPath, lockServiceSubPath)
+
+const (
+	lockRESTMethodLock        = "lock"
+	lockRESTMethodRLock       = "rlock"
+	lockRESTMethodUnlock      = "unlock"
+	lockRESTMethodRUnlock     = "runlock"
+	lockRESTMethodForceUnlock = "forceunlock"
+	lockRESTMethodExpired     = "expired"
 )
 
 // nameLockRequesterInfoPair is a helper type for lock maintenance
@@ -30,32 +41,30 @@ type nameLockRequesterInfoPair struct {
 	lri  lockRequesterInfo
 }
 
+type lockResponse struct {
+	Success bool
+}
+
 // Similar to removeEntry but only removes an entry only if the lock entry exists in map.
 func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
 	// Check if entry is still in map (could have been removed altogether by 'concurrent' (R)Unlock of last entry)
 	if lri, ok := l.lockMap[nlrip.name]; ok {
-		if !l.removeEntry(nlrip.name, nlrip.lri.UID, &lri) {
-			// Remove failed, in case it is a:
-			if nlrip.lri.Writer {
-				// Writer: this should never happen as the whole (mapped) entry should have been deleted
-				reqInfo := (&logger.ReqInfo{}).AppendTags("name", nlrip.name)
-				reqInfo.AppendTags("uid", nlrip.lri.UID)
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
-				logger.LogIf(ctx, errors.New("Lock maintenance failed to remove entry for write lock (should never happen)"))
-			} // Reader: this can happen if multiple read locks were active and
-			// the one we are looking for has been released concurrently (so it is fine).
-		} // Removal went okay, all is fine.
+		// Even if the entry exists, it may not be the same entry which was
+		// considered as expired, so we simply an attempt to remove it if its
+		// not possible there is nothing we need to do.
+		l.removeEntry(nlrip.name, nlrip.lri.UID, &lri)
 	}
 }
 
-// removeEntry either, based on the uid of the lock message, removes a single entry from the
-// lockRequesterInfo array or the whole array from the map (in case of a write lock or last read lock)
+// removeEntry based on the uid of the lock message, removes a single entry from the
+// lockRequesterInfo array or the whole array from the map (in case of a write lock
+// or last read lock)
 func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bool {
 	// Find correct entry to remove based on uid.
 	for index, entry := range *lri {
 		if entry.UID == uid {
 			if len(*lri) == 1 {
-				// Remove the (last) lock.
+				// Remove the write lock.
 				delete(l.lockMap, name)
 			} else {
 				// Remove the appropriate read lock.
@@ -65,6 +74,7 @@ func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bo
 			return true
 		}
 	}
+
 	// None found return false, perhaps entry removed in previous run.
 	return false
 }

cmd/lock-rest-server-common_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018, 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,10 +19,36 @@ package cmd
 import (
 	"os"
 	"reflect"
+	"sync"
 	"testing"
 	"time"
 )
 
+// Helper function to create a lock server for testing
+func createLockTestServer(t *testing.T) (string, *lockRESTServer, string) {
+	obj, fsDir, err := prepareFS()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+
+	locker := &lockRESTServer{
+		ll: localLocker{
+			mutex:           sync.Mutex{},
+			serviceEndpoint: "rpc-path",
+			lockMap:         make(map[string][]lockRequesterInfo),
+		},
+	}
+	creds := globalServerConfig.GetCredential()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return fsDir, locker, token
+}
+
 // Test function to remove lock entries from map only in case they still exist based on name & uid combination
 func TestLockRpcServerRemoveEntryIfExists(t *testing.T) {
 	testPath, locker, _ := createLockTestServer(t)

cmd/lock-rest-server.go

@@ -0,0 +1,336 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/gob"
+	"errors"
+	"math/rand"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/dsync"
+	"github.com/minio/minio/cmd/logger"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+const (
+	// Lock rpc server endpoint.
+	lockServiceSubPath = "/lock"
+
+	// Lock maintenance interval.
+	lockMaintenanceInterval = 1 * time.Minute
+
+	// Lock validity check interval.
+	lockValidityCheckInterval = 2 * time.Minute
+)
+
+// To abstract a node over network.
+type lockRESTServer struct {
+	ll localLocker
+}
+
+func (l *lockRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusForbidden)
+	w.Write([]byte(err.Error()))
+}
+
+// IsValid - To authenticate and verify the time difference.
+func (l *lockRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
+	if err := storageServerRequestValidate(r); err != nil {
+		l.writeErrorResponse(w, err)
+		return false
+	}
+	return true
+}
+
+// LockHandler - Acquires a lock.
+func (l *lockRESTServer) LockHandler(w http.ResponseWriter, r *http.Request) {
+	if !l.IsValid(w, r) {
+		l.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "Lock")
+
+	var lockArgs dsync.LockArgs
+	if r.ContentLength < 0 {
+		l.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	success, err := l.ll.Lock(lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+	resp := lockResponse{Success: success}
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(resp))
+	w.(http.Flusher).Flush()
+}
+
+// UnlockHandler - releases the acquired lock.
+func (l *lockRESTServer) UnlockHandler(w http.ResponseWriter, r *http.Request) {
+	if !l.IsValid(w, r) {
+		l.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "Unlock")
+
+	var lockArgs dsync.LockArgs
+	if r.ContentLength < 0 {
+		l.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	success, err := l.ll.Unlock(lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+	resp := lockResponse{Success: success}
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(resp))
+	w.(http.Flusher).Flush()
+}
+
+// LockHandler - Acquires an RLock.
+func (l *lockRESTServer) RLockHandler(w http.ResponseWriter, r *http.Request) {
+	if !l.IsValid(w, r) {
+		l.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "RLock")
+	var lockArgs dsync.LockArgs
+	if r.ContentLength < 0 {
+		l.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	success, err := l.ll.RLock(lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+	resp := lockResponse{Success: success}
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(resp))
+	w.(http.Flusher).Flush()
+}
+
+// RUnlockHandler - releases the acquired read lock.
+func (l *lockRESTServer) RUnlockHandler(w http.ResponseWriter, r *http.Request) {
+	if !l.IsValid(w, r) {
+		l.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "RUnlock")
+	var lockArgs dsync.LockArgs
+	if r.ContentLength < 0 {
+		l.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	success, err := l.ll.RUnlock(lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+	resp := lockResponse{Success: success}
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(resp))
+	w.(http.Flusher).Flush()
+}
+
+// ForceUnlockHandler - force releases the acquired lock.
+func (l *lockRESTServer) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
+	if !l.IsValid(w, r) {
+		l.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "ForceUnlock")
+
+	var lockArgs dsync.LockArgs
+	if r.ContentLength < 0 {
+		l.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	success, err := l.ll.ForceUnlock(lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+	resp := lockResponse{Success: success}
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(resp))
+	w.(http.Flusher).Flush()
+}
+
+// ExpiredHandler - query expired lock status.
+func (l *lockRESTServer) ExpiredHandler(w http.ResponseWriter, r *http.Request) {
+	if !l.IsValid(w, r) {
+		l.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "Expired")
+
+	var lockArgs dsync.LockArgs
+	if r.ContentLength < 0 {
+		l.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&lockArgs)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+	success := true
+	l.ll.mutex.Lock()
+	defer l.ll.mutex.Unlock()
+	// Lock found, proceed to verify if belongs to given uid.
+	if lri, ok := l.ll.lockMap[lockArgs.Resource]; ok {
+		// Check whether uid is still active
+		for _, entry := range lri {
+			if entry.UID == lockArgs.UID {
+				success = false // When uid found, lock is still active so return not expired.
+				break
+			}
+		}
+	}
+	// When we get here lock is no longer active due to either dsync.LockArgs.Resource
+	// being absent from map or uid not found for given dsync.LockArgs.Resource
+	resp := lockResponse{Success: success}
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(resp))
+	w.(http.Flusher).Flush()
+}
+
+// lockMaintenance loops over locks that have been active for some time and checks back
+// with the original server whether it is still alive or not
+//
+// Following logic inside ignores the errors generated for Dsync.Active operation.
+// - server at client down
+// - some network error (and server is up normally)
+//
+// We will ignore the error, and we will retry later to get a resolve on this lock
+func (l *lockRESTServer) lockMaintenance(interval time.Duration) {
+	l.ll.mutex.Lock()
+	// Get list of long lived locks to check for staleness.
+	nlripLongLived := getLongLivedLocks(l.ll.lockMap, interval)
+	l.ll.mutex.Unlock()
+
+	// Validate if long lived locks are indeed clean.
+	for _, nlrip := range nlripLongLived {
+		// Initialize client based on the long live locks.
+		host, err := xnet.ParseHost(nlrip.lri.Node)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			continue
+		}
+		c := newlockRESTClient(host)
+		if !c.connected {
+			continue
+		}
+
+		// Call back to original server verify whether the lock is still active (based on name & uid)
+		expired, _ := c.Expired(dsync.LockArgs{
+			UID:      nlrip.lri.UID,
+			Resource: nlrip.name,
+		})
+
+		// For successful response, verify if lock was indeed active or stale.
+		if expired {
+			// The lock is no longer active at server that originated
+			// the lock, attempt to remove the lock.
+			l.ll.mutex.Lock()
+			l.ll.removeEntryIfExists(nlrip) // Purge the stale entry if it exists.
+			l.ll.mutex.Unlock()
+		}
+
+		// Close the connection regardless of the call response.
+		c.Close()
+	}
+}
+
+// Start lock maintenance from all lock servers.
+func startLockMaintenance(lkSrv *lockRESTServer) {
+	// Initialize a new ticker with a minute between each ticks.
+	ticker := time.NewTicker(lockMaintenanceInterval)
+	// Stop the timer upon service closure and cleanup the go-routine.
+	defer ticker.Stop()
+
+	// Start with random sleep time, so as to avoid "synchronous checks" between servers
+	time.Sleep(time.Duration(rand.Float64() * float64(lockMaintenanceInterval)))
+	for {
+		// Verifies every minute for locks held more than 2 minutes.
+		select {
+		case <-GlobalServiceDoneCh:
+			return
+		case <-ticker.C:
+			lkSrv.lockMaintenance(lockValidityCheckInterval)
+		}
+	}
+}
+
+// registerLockRESTHandlers - register lock rest router.
+func registerLockRESTHandlers(router *mux.Router) {
+	subrouter := router.PathPrefix(lockRESTPath).Subrouter()
+	subrouter.Methods(http.MethodPost).Path("/" + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(globalLockServer.LockHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(globalLockServer.RLockHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(globalLockServer.UnlockHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(globalLockServer.RUnlockHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + lockRESTMethodForceUnlock).HandlerFunc(httpTraceHdrs(globalLockServer.ForceUnlockHandler))
+	subrouter.Methods(http.MethodPost).Path("/" + lockRESTMethodExpired).HandlerFunc(httpTraceAll(globalLockServer.ExpiredHandler))
+	router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
+
+	// Start lock maintenance from all lock servers.
+	go startLockMaintenance(globalLockServer)
+}

cmd/lock-rpc-client.go

@@ -1,114 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/tls"
-
-	"github.com/minio/dsync"
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-// LockRPCClient is authenticable lock RPC client compatible to dsync.NetLocker
-type LockRPCClient struct {
-	*RPCClient
-}
-
-// ServerAddr - dsync.NetLocker interface compatible method.
-func (lockRPC *LockRPCClient) ServerAddr() string {
-	url := lockRPC.ServiceURL()
-	return url.Host
-}
-
-// ServiceEndpoint - dsync.NetLocker interface compatible method.
-func (lockRPC *LockRPCClient) ServiceEndpoint() string {
-	url := lockRPC.ServiceURL()
-	return url.Path
-}
-
-// RLock calls read lock RPC.
-func (lockRPC *LockRPCClient) RLock(args dsync.LockArgs) (reply bool, err error) {
-	err = lockRPC.Call(lockServiceName+".RLock", &LockArgs{LockArgs: args}, &reply)
-	return reply, err
-}
-
-// Lock calls write lock RPC.
-func (lockRPC *LockRPCClient) Lock(args dsync.LockArgs) (reply bool, err error) {
-	err = lockRPC.Call(lockServiceName+".Lock", &LockArgs{LockArgs: args}, &reply)
-	return reply, err
-}
-
-// RUnlock calls read unlock RPC.
-func (lockRPC *LockRPCClient) RUnlock(args dsync.LockArgs) (reply bool, err error) {
-	err = lockRPC.Call(lockServiceName+".RUnlock", &LockArgs{LockArgs: args}, &reply)
-	return reply, err
-}
-
-// Unlock calls write unlock RPC.
-func (lockRPC *LockRPCClient) Unlock(args dsync.LockArgs) (reply bool, err error) {
-	err = lockRPC.Call(lockServiceName+".Unlock", &LockArgs{LockArgs: args}, &reply)
-	return reply, err
-}
-
-// ForceUnlock calls force unlock RPC.
-func (lockRPC *LockRPCClient) ForceUnlock(args dsync.LockArgs) (reply bool, err error) {
-	err = lockRPC.Call(lockServiceName+".ForceUnlock", &LockArgs{LockArgs: args}, &reply)
-	return reply, err
-}
-
-// Expired calls expired RPC.
-func (lockRPC *LockRPCClient) Expired(args dsync.LockArgs) (reply bool, err error) {
-	err = lockRPC.Call(lockServiceName+".Expired", &LockArgs{LockArgs: args}, &reply)
-	return reply, err
-}
-
-// NewLockRPCClient - returns new lock RPC client.
-func NewLockRPCClient(host *xnet.Host) (*LockRPCClient, error) {
-	scheme := "http"
-	if globalIsSSL {
-		scheme = "https"
-	}
-
-	serviceURL := &xnet.URL{
-		Scheme: scheme,
-		Host:   host.String(),
-		Path:   lockServicePath,
-	}
-
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: host.Name,
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	rpcClient, err := NewRPCClient(
-		RPCClientArgs{
-			NewAuthTokenFunc: newAuthToken,
-			RPCVersion:       globalRPCAPIVersion,
-			ServiceName:      lockServiceName,
-			ServiceURL:       serviceURL,
-			TLSConfig:        tlsConfig,
-		},
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return &LockRPCClient{rpcClient}, nil
-}

cmd/lock-rpc-server.go

@@ -1,196 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018, 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"math/rand"
-	"path"
-	"time"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/dsync"
-	"github.com/minio/minio/cmd/logger"
-	xrpc "github.com/minio/minio/cmd/rpc"
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-const (
-	// Lock rpc server endpoint.
-	lockServiceSubPath = "/lock"
-
-	// Lock rpc service name.
-	lockServiceName = "Dsync"
-
-	// Lock maintenance interval.
-	lockMaintenanceInterval = 1 * time.Minute
-
-	// Lock validity check interval.
-	lockValidityCheckInterval = 2 * time.Minute
-)
-
-var lockServicePath = path.Join(minioReservedBucketPath, lockServiceSubPath)
-
-// LockArgs represents arguments for any authenticated lock RPC call.
-type LockArgs struct {
-	AuthArgs
-	LockArgs dsync.LockArgs
-}
-
-// lockRPCReceiver is type for RPC handlers
-type lockRPCReceiver struct {
-	ll localLocker
-}
-
-// Lock - rpc handler for (single) write lock operation.
-func (l *lockRPCReceiver) Lock(args *LockArgs, reply *bool) (err error) {
-	*reply, err = l.ll.Lock(args.LockArgs)
-	return err
-}
-
-// Unlock - rpc handler for (single) write unlock operation.
-func (l *lockRPCReceiver) Unlock(args *LockArgs, reply *bool) (err error) {
-	*reply, err = l.ll.Unlock(args.LockArgs)
-	return err
-}
-
-// RLock - rpc handler for read lock operation.
-func (l *lockRPCReceiver) RLock(args *LockArgs, reply *bool) (err error) {
-	*reply, err = l.ll.RLock(args.LockArgs)
-	return err
-}
-
-// RUnlock - rpc handler for read unlock operation.
-func (l *lockRPCReceiver) RUnlock(args *LockArgs, reply *bool) (err error) {
-	*reply, err = l.ll.RUnlock(args.LockArgs)
-	return err
-}
-
-// ForceUnlock - rpc handler for force unlock operation.
-func (l *lockRPCReceiver) ForceUnlock(args *LockArgs, reply *bool) (err error) {
-	*reply, err = l.ll.ForceUnlock(args.LockArgs)
-	return err
-}
-
-// Expired - rpc handler for expired lock status.
-func (l *lockRPCReceiver) Expired(args *LockArgs, reply *bool) error {
-	l.ll.mutex.Lock()
-	defer l.ll.mutex.Unlock()
-	// Lock found, proceed to verify if belongs to given uid.
-	if lri, ok := l.ll.lockMap[args.LockArgs.Resource]; ok {
-		// Check whether uid is still active
-		for _, entry := range lri {
-			if entry.UID == args.LockArgs.UID {
-				*reply = false // When uid found, lock is still active so return not expired.
-				return nil     // When uid found *reply is set to true.
-			}
-		}
-	}
-	// When we get here lock is no longer active due to either args.LockArgs.Resource
-	// being absent from map or uid not found for given args.LockArgs.Resource
-	*reply = true
-	return nil
-}
-
-// lockMaintenance loops over locks that have been active for some time and checks back
-// with the original server whether it is still alive or not
-//
-// Following logic inside ignores the errors generated for Dsync.Active operation.
-// - server at client down
-// - some network error (and server is up normally)
-//
-// We will ignore the error, and we will retry later to get a resolve on this lock
-func (l *lockRPCReceiver) lockMaintenance(interval time.Duration) {
-	l.ll.mutex.Lock()
-	// Get list of long lived locks to check for staleness.
-	nlripLongLived := getLongLivedLocks(l.ll.lockMap, interval)
-	l.ll.mutex.Unlock()
-
-	// Validate if long lived locks are indeed clean.
-	for _, nlrip := range nlripLongLived {
-		// Initialize client based on the long live locks.
-		host, err := xnet.ParseHost(nlrip.lri.Node)
-		if err != nil {
-			logger.LogIf(context.Background(), err)
-			continue
-		}
-		c, err := NewLockRPCClient(host)
-		if err != nil {
-			logger.LogIf(context.Background(), err)
-			continue
-		}
-
-		// Call back to original server verify whether the lock is still active (based on name & uid)
-		expired, _ := c.Expired(dsync.LockArgs{
-			UID:      nlrip.lri.UID,
-			Resource: nlrip.name,
-		})
-
-		// Close the connection regardless of the call response.
-		c.Close()
-
-		// For successful response, verify if lock is indeed active or stale.
-		if expired {
-			// The lock is no longer active at server that originated the lock
-			// So remove the lock from the map.
-			l.ll.mutex.Lock()
-			l.ll.removeEntryIfExists(nlrip) // Purge the stale entry if it exists.
-			l.ll.mutex.Unlock()
-		}
-	}
-}
-
-// Start lock maintenance from all lock servers.
-func startLockMaintenance(lkSrv *lockRPCReceiver) {
-	// Initialize a new ticker with a minute between each ticks.
-	ticker := time.NewTicker(lockMaintenanceInterval)
-	// Stop the timer upon service closure and cleanup the go-routine.
-	defer ticker.Stop()
-
-	// Start with random sleep time, so as to avoid "synchronous checks" between servers
-	time.Sleep(time.Duration(rand.Float64() * float64(lockMaintenanceInterval)))
-	for {
-		// Verifies every minute for locks held more than 2minutes.
-		select {
-		case <-GlobalServiceDoneCh:
-			return
-		case <-ticker.C:
-			lkSrv.lockMaintenance(lockValidityCheckInterval)
-		}
-	}
-}
-
-// NewLockRPCServer - returns new lock RPC server.
-func NewLockRPCServer() (*xrpc.Server, error) {
-	rpcServer := xrpc.NewServer()
-	if err := rpcServer.RegisterName(lockServiceName, globalLockServer); err != nil {
-		return nil, err
-	}
-	return rpcServer, nil
-}
-
-// Register distributed NS lock handlers.
-func registerDistNSLockRouter(router *mux.Router) {
-	rpcServer, err := NewLockRPCServer()
-	logger.FatalIf(err, "Unable to initialize Lock RPC Server")
-
-	// Start lock maintenance from all lock servers.
-	go startLockMaintenance(globalLockServer)
-
-	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
-	subrouter.Path(lockServiceSubPath).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
-}

cmd/lock-rpc-server_test.go

@@ -1,538 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018, 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"os"
-	"runtime"
-	"sync"
-	"testing"
-
-	"github.com/minio/dsync"
-)
-
-// Helper function to test equality of locks (without taking timing info into account)
-func testLockEquality(lriLeft, lriRight []lockRequesterInfo) bool {
-	if len(lriLeft) != len(lriRight) {
-		return false
-	}
-
-	for i := 0; i < len(lriLeft); i++ {
-		if lriLeft[i].Writer != lriRight[i].Writer ||
-			lriLeft[i].Node != lriRight[i].Node ||
-			lriLeft[i].ServiceEndpoint != lriRight[i].ServiceEndpoint ||
-			lriLeft[i].UID != lriRight[i].UID {
-			return false
-		}
-	}
-	return true
-}
-
-// Helper function to create a lock server for testing
-func createLockTestServer(t *testing.T) (string, *lockRPCReceiver, string) {
-	obj, fsDir, err := prepareFS()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
-		t.Fatalf("unable initialize config file, %s", err)
-	}
-
-	locker := &lockRPCReceiver{
-		ll: localLocker{
-			mutex:           sync.Mutex{},
-			serviceEndpoint: "rpc-path",
-			lockMap:         make(map[string][]lockRequesterInfo),
-		},
-	}
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return fsDir, locker, token
-}
-
-// Test Lock functionality
-func TestLockRpcServerLock(t *testing.T) {
-	testPath, locker, token := createLockTestServer(t)
-	defer os.RemoveAll(testPath)
-
-	la := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "0123-4567",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	// Claim a lock
-	var result bool
-	err := locker.Lock(&la, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else {
-		if !result {
-			t.Errorf("Expected %#v, got %#v", true, result)
-		} else {
-			gotLri := locker.ll.lockMap["name"]
-			expectedLri := []lockRequesterInfo{
-				{
-					Writer:          true,
-					Node:            "node",
-					ServiceEndpoint: "rpc-path",
-					UID:             "0123-4567",
-				},
-			}
-			if !testLockEquality(expectedLri, gotLri) {
-				t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
-			}
-		}
-	}
-
-	// Try to claim same lock again (will fail)
-	la2 := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "89ab-cdef",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	err = locker.Lock(&la2, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else {
-		if result {
-			t.Errorf("Expected %#v, got %#v", false, result)
-		}
-	}
-}
-
-// Test Unlock functionality
-func TestLockRpcServerUnlock(t *testing.T) {
-	testPath, locker, token := createLockTestServer(t)
-	defer os.RemoveAll(testPath)
-
-	la := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "0123-4567",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	// First test return of error when attempting to unlock a lock that does not exist
-	var result bool
-	err := locker.Unlock(&la, &result)
-	if err == nil {
-		t.Errorf("Expected error, got %#v", nil)
-	}
-
-	// Create lock (so that we can release)
-	err = locker.Lock(&la, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else if !result {
-		t.Errorf("Expected %#v, got %#v", true, result)
-	}
-
-	// Finally test successful release of lock
-	err = locker.Unlock(&la, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else {
-		if !result {
-			t.Errorf("Expected %#v, got %#v", true, result)
-		} else {
-			gotLri := locker.ll.lockMap["name"]
-			expectedLri := []lockRequesterInfo(nil)
-			if !testLockEquality(expectedLri, gotLri) {
-				t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
-			}
-		}
-	}
-}
-
-// Test RLock functionality
-func TestLockRpcServerRLock(t *testing.T) {
-	testPath, locker, token := createLockTestServer(t)
-	defer os.RemoveAll(testPath)
-
-	la := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "0123-4567",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	// Claim a lock
-	var result bool
-	err := locker.RLock(&la, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else {
-		if !result {
-			t.Errorf("Expected %#v, got %#v", true, result)
-		} else {
-			gotLri := locker.ll.lockMap["name"]
-			expectedLri := []lockRequesterInfo{
-				{
-					Writer:          false,
-					Node:            "node",
-					ServiceEndpoint: "rpc-path",
-					UID:             "0123-4567",
-				},
-			}
-			if !testLockEquality(expectedLri, gotLri) {
-				t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
-			}
-		}
-	}
-
-	// Try to claim same again (will succeed)
-	la2 := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "89ab-cdef",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	err = locker.RLock(&la2, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else {
-		if !result {
-			t.Errorf("Expected %#v, got %#v", true, result)
-		}
-	}
-}
-
-// Test RUnlock functionality
-func TestLockRpcServerRUnlock(t *testing.T) {
-	testPath, locker, token := createLockTestServer(t)
-	defer os.RemoveAll(testPath)
-
-	la := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "0123-4567",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	// First test return of error when attempting to unlock a read-lock that does not exist
-	var result bool
-	err := locker.Unlock(&la, &result)
-	if err == nil {
-		t.Errorf("Expected error, got %#v", nil)
-	}
-
-	// Create first lock ... (so that we can release)
-	err = locker.RLock(&la, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else if !result {
-		t.Errorf("Expected %#v, got %#v", true, result)
-	}
-
-	// Try to claim same again (will succeed)
-	la2 := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "89ab-cdef",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	// ... and create a second lock on same resource
-	err = locker.RLock(&la2, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else if !result {
-		t.Errorf("Expected %#v, got %#v", true, result)
-	}
-
-	// Test successful release of first read lock
-	err = locker.RUnlock(&la, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else {
-		if !result {
-			t.Errorf("Expected %#v, got %#v", true, result)
-		} else {
-			gotLri := locker.ll.lockMap["name"]
-			expectedLri := []lockRequesterInfo{
-				{
-					Writer:          false,
-					Node:            "node",
-					ServiceEndpoint: "rpc-path",
-					UID:             "89ab-cdef",
-				},
-			}
-			if !testLockEquality(expectedLri, gotLri) {
-				t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
-			}
-
-		}
-	}
-
-	// Finally test successful release of second (and last) read lock
-	err = locker.RUnlock(&la2, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else {
-		if !result {
-			t.Errorf("Expected %#v, got %#v", true, result)
-		} else {
-			gotLri := locker.ll.lockMap["name"]
-			expectedLri := []lockRequesterInfo(nil)
-			if !testLockEquality(expectedLri, gotLri) {
-				t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
-			}
-		}
-	}
-}
-
-// Test ForceUnlock functionality
-func TestLockRpcServerForceUnlock(t *testing.T) {
-	testPath, locker, token := createLockTestServer(t)
-	defer os.RemoveAll(testPath)
-
-	laForce := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "1234-5678",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	// First test that UID should be empty
-	var result bool
-	err := locker.ForceUnlock(&laForce, &result)
-	if err == nil {
-		t.Errorf("Expected error, got %#v", nil)
-	}
-
-	// Then test force unlock of a lock that does not exist (not returning an error)
-	laForce.LockArgs.UID = ""
-	err = locker.ForceUnlock(&laForce, &result)
-	if err != nil {
-		t.Errorf("Expected no error, got %#v", err)
-	}
-
-	la := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "0123-4567",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	// Create lock ... (so that we can force unlock)
-	err = locker.Lock(&la, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else if !result {
-		t.Errorf("Expected %#v, got %#v", true, result)
-	}
-
-	// Forcefully unlock the lock (not returning an error)
-	err = locker.ForceUnlock(&laForce, &result)
-	if err != nil {
-		t.Errorf("Expected no error, got %#v", err)
-	}
-
-	// Try to get lock again (should be granted)
-	err = locker.Lock(&la, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else if !result {
-		t.Errorf("Expected %#v, got %#v", true, result)
-	}
-
-	// Finally forcefully unlock the lock once again
-	err = locker.ForceUnlock(&laForce, &result)
-	if err != nil {
-		t.Errorf("Expected no error, got %#v", err)
-	}
-}
-
-// Test Expired functionality
-func TestLockRpcServerExpired(t *testing.T) {
-	testPath, locker, token := createLockTestServer(t)
-	defer os.RemoveAll(testPath)
-
-	la := LockArgs{
-		AuthArgs: AuthArgs{
-			Token:       token,
-			RPCVersion:  globalRPCAPIVersion,
-			RequestTime: UTCNow(),
-		},
-		LockArgs: dsync.LockArgs{
-			UID:             "0123-4567",
-			Resource:        "name",
-			ServerAddr:      "node",
-			ServiceEndpoint: "rpc-path",
-		}}
-
-	// Unknown lock at server will return expired = true
-	var expired bool
-	err := locker.Expired(&la, &expired)
-	if err != nil {
-		t.Errorf("Expected no error, got %#v", err)
-	} else {
-		if !expired {
-			t.Errorf("Expected %#v, got %#v", true, expired)
-		}
-	}
-
-	// Create lock (so that we can test that it is not expired)
-	var result bool
-	err = locker.Lock(&la, &result)
-	if err != nil {
-		t.Errorf("Expected %#v, got %#v", nil, err)
-	} else if !result {
-		t.Errorf("Expected %#v, got %#v", true, result)
-	}
-
-	err = locker.Expired(&la, &expired)
-	if err != nil {
-		t.Errorf("Expected no error, got %#v", err)
-	} else {
-		if expired {
-			t.Errorf("Expected %#v, got %#v", false, expired)
-		}
-	}
-}
-
-// Test initialization of lock server.
-func TestLockServerInit(t *testing.T) {
-	if runtime.GOOS == globalWindowsOSName {
-		return
-	}
-
-	obj, fsDir, err := prepareFS()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(fsDir)
-	if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
-		t.Fatalf("unable initialize config file, %s", err)
-	}
-
-	currentIsDistXL := globalIsDistXL
-	currentLockServer := globalLockServer
-	defer func() {
-		globalIsDistXL = currentIsDistXL
-		globalLockServer = currentLockServer
-	}()
-
-	case1Endpoints := mustGetNewEndpointList(
-		"http://localhost:9000/mnt/disk1",
-		"http://1.1.1.2:9000/mnt/disk2",
-		"http://1.1.2.1:9000/mnt/disk3",
-		"http://1.1.2.2:9000/mnt/disk4",
-	)
-	for i := range case1Endpoints {
-		if case1Endpoints[i].Host == "localhost:9000" {
-			case1Endpoints[i].IsLocal = true
-		}
-	}
-
-	case2Endpoints := mustGetNewEndpointList(
-		"http://localhost:9000/mnt/disk1",
-		"http://localhost:9000/mnt/disk2",
-		"http://1.1.2.1:9000/mnt/disk3",
-		"http://1.1.2.2:9000/mnt/disk4",
-	)
-	for i := range case2Endpoints {
-		if case2Endpoints[i].Host == "localhost:9000" {
-			case2Endpoints[i].IsLocal = true
-		}
-	}
-
-	globalMinioHost = ""
-	testCases := []struct {
-		isDistXL  bool
-		endpoints EndpointList
-	}{
-		// Test - 1 one lock server initialized.
-		{true, case1Endpoints},
-		// Test - similar endpoint hosts should
-		// converge to single lock server
-		// initialized.
-		{true, case2Endpoints},
-	}
-
-	// Validates lock server initialization.
-	for i, testCase := range testCases {
-		globalIsDistXL = testCase.isDistXL
-		globalLockServer = nil
-		_, _ = newDsyncNodes(testCase.endpoints)
-		if globalLockServer == nil && testCase.isDistXL {
-			t.Errorf("Test %d: Expected initialized lock RPC receiver, but got uninitialized", i+1)
-		}
-	}
-}

cmd/logger/target/console/console.go

@@ -45,12 +45,13 @@ func (c *Target) Send(e interface{}) error {
 		return nil
 	}
 
-	trace := make([]string, len(entry.Trace.Source))
+	traceLength := len(entry.Trace.Source)
+	trace := make([]string, traceLength)
 
 	// Add a sequence number and formatting for each stack trace
 	// No formatting is required for the first entry
 	for i, element := range entry.Trace.Source {
-		trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
+		trace[i] = fmt.Sprintf("%8v: %s", traceLength-i, element)
 	}
 
 	tagString := ""

cmd/main.go

@@ -32,24 +32,28 @@ var globalFlags = []cli.Flag{
 	cli.StringFlag{
 		Name:  "config-dir, C",
 		Value: defaultConfigDir.Get(),
-		Usage: "[DEPRECATED] Path to legacy configuration directory.",
+		Usage: "[DEPRECATED] path to legacy configuration directory",
 	},
 	cli.StringFlag{
 		Name:  "certs-dir, S",
 		Value: defaultCertsDir.Get(),
-		Usage: "Path to certs directory.",
+		Usage: "path to certs directory",
 	},
 	cli.BoolFlag{
 		Name:  "quiet",
-		Usage: "Disable startup information.",
+		Usage: "disable startup information",
 	},
 	cli.BoolFlag{
 		Name:  "anonymous",
-		Usage: "Hide sensitive information from logging.",
+		Usage: "hide sensitive information from logging",
 	},
 	cli.BoolFlag{
 		Name:  "json",
-		Usage: "Output server logs and startup information in json format.",
+		Usage: "output server logs and startup information in json format",
+	},
+	cli.BoolFlag{
+		Name:  "compat",
+		Usage: "trade off performance for S3 compatibility",
 	},
 }
 
@@ -118,7 +122,7 @@ func newApp(name string) *cli.App {
 	// Set up app.
 	cli.HelpFlag = cli.BoolFlag{
 		Name:  "help, h",
-		Usage: "Show help.",
+		Usage: "show help",
 	}
 
 	app := cli.NewApp()

cmd/merge-walk-pool.go

@@ -0,0 +1,143 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"reflect"
+	"sync"
+	"time"
+)
+
+const (
+	globalMergeLookupTimeout = time.Minute * 1 // 1 minutes.
+)
+
+// mergeWalk - represents the go routine that does the merge walk.
+type mergeWalk struct {
+	entryChs   []FileInfoCh
+	endWalkCh  chan struct{}   // To signal when mergeWalk go-routine should end.
+	endTimerCh chan<- struct{} // To signal when timer go-routine should end.
+}
+
+// MergeWalkPool - pool of mergeWalk go routines.
+// A mergeWalk is added to the pool by Set() and removed either by
+// doing a Release() or if the concerned timer goes off.
+// mergeWalkPool's purpose is to maintain active mergeWalk go-routines in a map so that
+// it can be looked up across related list calls.
+type MergeWalkPool struct {
+	pool    map[listParams][]mergeWalk
+	timeOut time.Duration
+	lock    *sync.Mutex
+}
+
+// NewMergeWalkPool - initialize new tree walk pool.
+func NewMergeWalkPool(timeout time.Duration) *MergeWalkPool {
+	tPool := &MergeWalkPool{
+		pool:    make(map[listParams][]mergeWalk),
+		timeOut: timeout,
+		lock:    &sync.Mutex{},
+	}
+	return tPool
+}
+
+// Release - selects a mergeWalk from the pool based on the input
+// listParams, removes it from the pool, and returns the MergeWalkResult
+// channel.
+// Returns nil if listParams does not have an asccociated mergeWalk.
+func (t MergeWalkPool) Release(params listParams) ([]FileInfoCh, chan struct{}) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	walks, ok := t.pool[params] // Pick the valid walks.
+	if ok {
+		if len(walks) > 0 {
+			// Pop out the first valid walk entry.
+			walk := walks[0]
+			walks = walks[1:]
+			if len(walks) > 0 {
+				t.pool[params] = walks
+			} else {
+				delete(t.pool, params)
+			}
+			walk.endTimerCh <- struct{}{}
+			return walk.entryChs, walk.endWalkCh
+		}
+	}
+	// Release return nil if params not found.
+	return nil, nil
+}
+
+// Set - adds a mergeWalk to the mergeWalkPool.
+// Also starts a timer go-routine that ends when:
+// 1) time.After() expires after t.timeOut seconds.
+//    The expiration is needed so that the mergeWalk go-routine resources are freed after a timeout
+//    if the S3 client does only partial listing of objects.
+// 2) Relase() signals the timer go-routine to end on endTimerCh.
+//    During listing the timer should not timeout and end the mergeWalk go-routine, hence the
+//    timer go-routine should be ended.
+func (t MergeWalkPool) Set(params listParams, resultChs []FileInfoCh, endWalkCh chan struct{}) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	// Should be a buffered channel so that Release() never blocks.
+	endTimerCh := make(chan struct{}, 1)
+
+	walkInfo := mergeWalk{
+		entryChs:   resultChs,
+		endWalkCh:  endWalkCh,
+		endTimerCh: endTimerCh,
+	}
+
+	// Append new walk info.
+	t.pool[params] = append(t.pool[params], walkInfo)
+
+	// Timer go-routine which times out after t.timeOut seconds.
+	go func(endTimerCh <-chan struct{}, walkInfo mergeWalk) {
+		select {
+		// Wait until timeOut
+		case <-time.After(t.timeOut):
+			// Timeout has expired. Remove the mergeWalk from mergeWalkPool and
+			// end the mergeWalk go-routine.
+			t.lock.Lock()
+			walks, ok := t.pool[params]
+			if ok {
+				// Trick of filtering without allocating
+				// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
+				nwalks := walks[:0]
+				// Look for walkInfo, remove it from the walks list.
+				for _, walk := range walks {
+					if !reflect.DeepEqual(walk, walkInfo) {
+						nwalks = append(nwalks, walk)
+					}
+				}
+				if len(nwalks) == 0 {
+					// No more mergeWalk go-routines associated with listParams
+					// hence remove map entry.
+					delete(t.pool, params)
+				} else {
+					// There are more mergeWalk go-routines associated with listParams
+					// hence save the list in the map.
+					t.pool[params] = nwalks
+				}
+			}
+			// Signal the mergeWalk go-routine to die.
+			close(endWalkCh)
+			t.lock.Unlock()
+		case <-endTimerCh:
+			return
+		}
+	}(endTimerCh, walkInfo)
+}

cmd/merge-walk-pool_test.go

@@ -0,0 +1,103 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"testing"
+	"time"
+)
+
+// Test if tree walker go-routine is removed from the pool after timeout
+// and that is available in the pool before the timeout.
+func TestMergeWalkPoolBasic(t *testing.T) {
+	// Create a treeWalkPool
+	tw := NewMergeWalkPool(1 * time.Second)
+
+	// Create sample params
+	params := listParams{
+		bucket: "test-bucket",
+	}
+
+	endWalkCh := make(chan struct{})
+	// Add a treeWalk to the pool
+	tw.Set(params, []FileInfoCh{}, endWalkCh)
+
+	// Wait for treeWalkPool timeout to happen
+	<-time.After(2 * time.Second)
+	if c1, _ := tw.Release(params); c1 != nil {
+		t.Error("treeWalk go-routine must have been freed")
+	}
+
+	// Add the treeWalk back to the pool
+	endWalkCh = make(chan struct{})
+	tw.Set(params, []FileInfoCh{}, endWalkCh)
+
+	// Release the treeWalk before timeout
+	select {
+	case <-time.After(1 * time.Second):
+		break
+	default:
+		if c1, _ := tw.Release(params); c1 == nil {
+			t.Error("treeWalk go-routine got freed before timeout")
+		}
+	}
+}
+
+// Test if multiple merge walkers for the same listParams are managed as expected by the pool.
+func TestManyMergeWalksSameParam(t *testing.T) {
+	// Create a treeWalkPool.
+	tw := NewMergeWalkPool(5 * time.Second)
+
+	// Create sample params.
+	params := listParams{
+		bucket: "test-bucket",
+	}
+
+	select {
+	// This timeout is an upper-bound. This is started
+	// before the first treeWalk go-routine's timeout period starts.
+	case <-time.After(5 * time.Second):
+		break
+	default:
+		// Create many treeWalk go-routines for the same params.
+		for i := 0; i < 10; i++ {
+			endWalkCh := make(chan struct{})
+			walkChs := make([]FileInfoCh, 0)
+			tw.Set(params, walkChs, endWalkCh)
+		}
+
+		tw.lock.Lock()
+		if walks, ok := tw.pool[params]; ok {
+			if len(walks) != 10 {
+				t.Error("There aren't as many walks as were Set")
+			}
+		}
+		tw.lock.Unlock()
+		for i := 0; i < 10; i++ {
+			tw.lock.Lock()
+			if walks, ok := tw.pool[params]; ok {
+				// Before ith Release we should have 10-i treeWalk go-routines.
+				if 10-i != len(walks) {
+					t.Error("There aren't as many walks as were Set")
+				}
+			}
+			tw.lock.Unlock()
+			tw.Release(params)
+		}
+	}
+
+}

cmd/namespace-lock.go

@@ -38,7 +38,7 @@ import (
 var globalNSMutex *nsLockMap
 
 // Global lock server one per server.
-var globalLockServer *lockRPCReceiver
+var globalLockServer *lockRESTServer
 
 // Instance of dsync for distributed clients.
 var globalDsync *dsync.Dsync
@@ -66,7 +66,7 @@ func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int)
 		if endpoint.IsLocal {
 			myNode = len(clnts)
 
-			receiver := &lockRPCReceiver{
+			receiver := &lockRESTServer{
 				ll: localLocker{
 					serverAddr:      endpoint.Host,
 					serviceEndpoint: lockServicePath,
@@ -79,8 +79,7 @@ func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int)
 		} else {
 			host, err := xnet.ParseHost(endpoint.Host)
 			logger.FatalIf(err, "Unable to parse Lock RPC Host")
-			locker, err = NewLockRPCClient(host)
-			logger.FatalIf(err, "Unable to initialize Lock RPC Client")
+			locker = newlockRESTClient(host)
 		}
 
 		clnts = append(clnts, locker)

cmd/naughty-disk_test.go

@@ -111,11 +111,18 @@ func (d *naughtyDisk) DeleteVol(volume string) (err error) {
 	return d.disk.DeleteVol(volume)
 }
 
-func (d *naughtyDisk) ListDir(volume, path string, count int) (entries []string, err error) {
+func (d *naughtyDisk) Walk(volume, path, marker string, recursive bool, leafFile string, readMetadataFn readMetadataFunc, endWalkCh chan struct{}) (chan FileInfo, error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.Walk(volume, path, marker, recursive, leafFile, readMetadataFn, endWalkCh)
+}
+
+func (d *naughtyDisk) ListDir(volume, path string, count int, leafFile string) (entries []string, err error) {
 	if err := d.calcError(); err != nil {
 		return []string{}, err
 	}
-	return d.disk.ListDir(volume, path, count)
+	return d.disk.ListDir(volume, path, count, leafFile)
 }
 
 func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
@@ -167,11 +174,19 @@ func (d *naughtyDisk) DeleteFile(volume string, path string) (err error) {
 	return d.disk.DeleteFile(volume, path)
 }
 
-func (d *naughtyDisk) WriteAll(volume string, path string, buf []byte) (err error) {
+func (d *naughtyDisk) DeleteFileBulk(volume string, paths []string) ([]error, error) {
+	errs := make([]error, len(paths))
+	for idx, path := range paths {
+		errs[idx] = d.disk.DeleteFile(volume, path)
+	}
+	return errs, nil
+}
+
+func (d *naughtyDisk) WriteAll(volume string, path string, reader io.Reader) (err error) {
 	if err := d.calcError(); err != nil {
 		return err
 	}
-	return d.disk.WriteAll(volume, path, buf)
+	return d.disk.WriteAll(volume, path, reader)
 }
 
 func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error) {

cmd/net.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net"
@@ -27,9 +26,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
-	"time"
 
-	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 )
@@ -102,40 +99,7 @@ func getHostIP(host string) (ipList set.StringSet, err error) {
 	var ips []net.IP
 
 	if ips, err = net.LookupIP(host); err != nil {
-		// return err if not Docker or Kubernetes
-		// We use IsDocker() method to check for Docker Swarm environment
-		// as there is no reliable way to clearly identify Swarm from
-		// Docker environment.
-		if !IsDocker() && !IsKubernetes() {
-			return ipList, err
-		}
-
-		// channel to indicate completion of host resolution
-		doneCh := make(chan struct{})
-		// Indicate retry routine to exit cleanly, upon this function return.
-		defer close(doneCh)
-		// Mark the starting time
-		startTime := time.Now()
-		// wait for hosts to resolve in exponentialbackoff manner
-		for range newRetryTimerSimple(doneCh) {
-			// Retry infinitely on Kubernetes and Docker swarm.
-			// This is needed as the remote hosts are sometime
-			// not available immediately.
-			if ips, err = net.LookupIP(host); err == nil {
-				break
-			}
-			// time elapsed
-			timeElapsed := time.Since(startTime)
-			// log error only if more than 1s elapsed
-			if timeElapsed > time.Second {
-				// log the message to console about the host not being
-				// resolveable.
-				reqInfo := (&logger.ReqInfo{}).AppendTags("host", host)
-				reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", ""))
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
-				logger.LogIf(ctx, err)
-			}
-		}
+		return ipList, err
 	}
 
 	ipList = set.NewStringSet()

cmd/notification.go

@@ -501,7 +501,6 @@ func (sys *NotificationSys) RemoteTargetExist(bucketName string, targetID event.
 
 // ListenBucketNotificationArgs - listen bucket notification RPC arguments.
 type ListenBucketNotificationArgs struct {
-	AuthArgs   `json:"-"`
 	BucketName string         `json:"-"`
 	EventNames []event.Name   `json:"eventNames"`
 	Pattern    string         `json:"pattern"`

cmd/object-api-common.go

@@ -116,7 +116,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
 		}
 
 		// If it's a directory, list and call delFunc() for each entry.
-		entries, err := storage.ListDir(volume, entryPath, -1)
+		entries, err := storage.ListDir(volume, entryPath, -1, "")
 		// If entryPath prefix never existed, safe to ignore.
 		if err == errFileNotFound {
 			return nil
@@ -144,6 +144,78 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
 	return err
 }
 
+// Cleanup objects in bulk and recursively: each object will have a list of sub-files to delete in the backend
+func cleanupObjectsBulk(ctx context.Context, storage StorageAPI, volume string, objsPaths []string, errs []error) ([]error, error) {
+	// The list of files in disk to delete
+	var filesToDelete []string
+	// Map files to delete to the passed objsPaths
+	var filesToDeleteObjsIndexes []int
+
+	// Traverse and return the list of sub entries
+	var traverse func(string) ([]string, error)
+	traverse = func(entryPath string) ([]string, error) {
+		var output = make([]string, 0)
+		if !hasSuffix(entryPath, slashSeparator) {
+			output = append(output, entryPath)
+			return output, nil
+		}
+		entries, err := storage.ListDir(volume, entryPath, -1, "")
+		if err != nil {
+			if err == errFileNotFound {
+				return nil, nil
+			}
+			return nil, err
+		}
+
+		for _, entry := range entries {
+			subEntries, err := traverse(pathJoin(entryPath, entry))
+			if err != nil {
+				return nil, err
+			}
+			output = append(output, subEntries...)
+		}
+		return output, nil
+	}
+
+	// Find and collect the list of files to remove associated
+	// to the passed objects paths
+	for idx, objPath := range objsPaths {
+		if errs[idx] != nil {
+			continue
+		}
+		output, err := traverse(objPath)
+		if err != nil {
+			errs[idx] = err
+			continue
+		} else {
+			errs[idx] = nil
+		}
+		filesToDelete = append(filesToDelete, output...)
+		for i := 0; i < len(output); i++ {
+			filesToDeleteObjsIndexes = append(filesToDeleteObjsIndexes, idx)
+		}
+	}
+
+	// Reverse the list so remove can succeed
+	reverseStringSlice(filesToDelete)
+
+	dErrs, err := storage.DeleteFileBulk(volume, filesToDelete)
+	if err != nil {
+		return nil, err
+	}
+
+	// Map files deletion errors to the correspondent objects
+	for i := range dErrs {
+		if dErrs[i] != nil {
+			if errs[filesToDeleteObjsIndexes[i]] != nil {
+				errs[filesToDeleteObjsIndexes[i]] = dErrs[i]
+			}
+		}
+	}
+
+	return errs, nil
+}
+
 // Removes notification.xml for a given bucket, only used during DeleteBucket.
 func removeNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
 	// Verify bucket is valid.
@@ -162,7 +234,7 @@ func removeListenerConfig(ctx context.Context, objAPI ObjectLayer, bucket string
 	return objAPI.DeleteObject(ctx, minioMetaBucket, lcPath)
 }
 
-func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
+func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
 	if err := checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, obj); err != nil {
 		return loi, err
 	}
@@ -203,7 +275,7 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 	walkResultCh, endWalkCh := tpool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
-		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
+		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, endWalkCh)
 	}
 
 	var objInfos []ObjectInfo
@@ -218,14 +290,6 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 			eof = true
 			break
 		}
-		// For any walk error return right away.
-		if walkResult.err != nil {
-			// File not found is a valid case.
-			if walkResult.err == errFileNotFound {
-				continue
-			}
-			return loi, toObjectErr(walkResult.err, bucket, prefix)
-		}
 
 		var objInfo ObjectInfo
 		var err error
@@ -235,6 +299,14 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 				if err == nil {
 					break
 				}
+				if err == errFileNotFound {
+					err = nil
+					objInfo = ObjectInfo{
+						Bucket: bucket,
+						Name:   walkResult.entry,
+						IsDir:  true,
+					}
+				}
 			}
 		} else {
 			objInfo, err = getObjInfo(ctx, bucket, walkResult.entry)

cmd/object-api-errors.go

@@ -27,6 +27,22 @@ import (
 // underlying storage layer.
 func toObjectErr(err error, params ...string) error {
 	switch err {
+	case errDiskNotFound:
+		switch len(params) {
+		case 1:
+			err = BucketNotFound{Bucket: params[0]}
+		case 2:
+			err = ObjectNotFound{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		case 3:
+			err = InvalidUploadID{
+				Bucket:   params[0],
+				Object:   params[1],
+				UploadID: params[2],
+			}
+		}
 	case errVolumeNotFound:
 		if len(params) >= 1 {
 			err = BucketNotFound{Bucket: params[0]}
@@ -41,6 +57,8 @@ func toObjectErr(err error, params ...string) error {
 		}
 	case errDiskFull:
 		err = StorageFull{}
+	case errTooManyOpenFiles:
+		err = SlowDown{}
 	case errFileAccessDenied:
 		if len(params) >= 2 {
 			err = PrefixAccessDenied{
@@ -63,11 +81,18 @@ func toObjectErr(err error, params ...string) error {
 			}
 		}
 	case errFileNotFound:
-		if len(params) >= 2 {
+		switch len(params) {
+		case 2:
 			err = ObjectNotFound{
 				Bucket: params[0],
 				Object: params[1],
 			}
+		case 3:
+			err = InvalidUploadID{
+				Bucket:   params[0],
+				Object:   params[1],
+				UploadID: params[2],
+			}
 		}
 	case errFileNameTooLong:
 		if len(params) >= 2 {
@@ -114,6 +139,13 @@ func (e StorageFull) Error() string {
 	return "Storage reached its minimum free disk threshold."
 }
 
+// SlowDown  too many file descriptors open or backend busy .
+type SlowDown struct{}
+
+func (e SlowDown) Error() string {
+	return "Please reduce your request rate"
+}
+
 // InsufficientReadQuorum storage cannot satisfy quorum for read operation.
 type InsufficientReadQuorum struct{}
 
@@ -321,6 +353,8 @@ func (e MalformedUploadID) Error() string {
 
 // InvalidUploadID invalid upload id.
 type InvalidUploadID struct {
+	Bucket   string
+	Object   string
 	UploadID string
 }
 

cmd/object-api-interface.go

@@ -73,6 +73,7 @@ type ObjectLayer interface {
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
+	DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error)
 
 	// Multipart operations.
 	ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)

cmd/object-api-listobjects_test.go

@@ -35,11 +35,13 @@ func TestListObjects(t *testing.T) {
 }
 
 // Unit test for ListObjects in general.
-func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
-
+func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
+	t, _ := t1.(*testing.T)
 	testBuckets := []string{
 		// This bucket is used for testing ListObject operations.
 		"test-bucket-list-object",
+		// This bucket will be tested with empty directories
+		"test-bucket-empty-dir",
 		// Will not store any objects in this bucket,
 		// Its to test ListObjects on an empty bucket.
 		"empty-bucket",
@@ -53,24 +55,27 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 	var err error
 	testObjects := []struct {
-		name    string
-		content string
-		meta    map[string]string
+		parentBucket string
+		name         string
+		content      string
+		meta         map[string]string
 	}{
-		{"Asia-maps.png", "asis-maps", map[string]string{"content-type": "image/png"}},
-		{"Asia/India/India-summer-photos-1", "contentstring", nil},
-		{"Asia/India/Karnataka/Bangalore/Koramangala/pics", "contentstring", nil},
-		{"newPrefix0", "newPrefix0", nil},
-		{"newPrefix1", "newPrefix1", nil},
-		{"newzen/zen/recurse/again/again/again/pics", "recurse", nil},
-		{"obj0", "obj0", nil},
-		{"obj1", "obj1", nil},
-		{"obj2", "obj2", nil},
-		{"z-empty-dir/", "", nil},
+		{testBuckets[0], "Asia-maps.png", "asis-maps", map[string]string{"content-type": "image/png"}},
+		{testBuckets[0], "Asia/India/India-summer-photos-1", "contentstring", nil},
+		{testBuckets[0], "Asia/India/Karnataka/Bangalore/Koramangala/pics", "contentstring", nil},
+		{testBuckets[0], "newPrefix0", "newPrefix0", nil},
+		{testBuckets[0], "newPrefix1", "newPrefix1", nil},
+		{testBuckets[0], "newzen/zen/recurse/again/again/again/pics", "recurse", nil},
+		{testBuckets[0], "obj0", "obj0", nil},
+		{testBuckets[0], "obj1", "obj1", nil},
+		{testBuckets[0], "obj2", "obj2", nil},
+		{testBuckets[1], "obj1", "obj1", nil},
+		{testBuckets[1], "obj2", "obj2", nil},
+		{testBuckets[1], "temporary/0/", "", nil},
 	}
 	for _, object := range testObjects {
 		md5Bytes := md5.Sum([]byte(object.content))
-		_, err = obj.PutObject(context.Background(), testBuckets[0], object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
+		_, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
 			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{UserDefined: object.meta})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
@@ -96,7 +101,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-1.
@@ -193,7 +197,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-10.
@@ -205,7 +208,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-11.
@@ -215,7 +217,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-12.
@@ -224,7 +225,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-13.
@@ -238,7 +238,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-14.
@@ -255,7 +254,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-15.
@@ -270,7 +268,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-16.
@@ -284,7 +281,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-17.
@@ -371,6 +367,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
 			},
+			Prefixes: []string{"Asia/"},
 		},
 		// ListObjectsResult-26.
 		// prefix = "new" and delimiter is set in the testCase.(testCase 58).
@@ -380,6 +377,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "newPrefix0"},
 				{Name: "newPrefix1"},
 			},
+			Prefixes: []string{"newzen/"},
 		},
 		// ListObjectsResult-27.
 		// Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59).
@@ -388,6 +386,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			Objects: []ObjectInfo{
 				{Name: "Asia/India/India-summer-photos-1"},
 			},
+			Prefixes: []string{"Asia/India/Karnataka/"},
 		},
 		// ListObjectsResult-28.
 		// Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60).
@@ -400,6 +399,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj1"},
 				{Name: "obj2"},
 			},
+			Prefixes: []string{"newzen/"},
 		},
 		// ListObjectsResult-29.
 		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimeter set, (testCase 61).
@@ -412,6 +412,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj1"},
 				{Name: "obj2"},
 			},
+			Prefixes: []string{"newzen/"},
 		},
 		// ListObjectsResult-30.
 		// Prefix and Delimiter is set to '/', (testCase 62).
@@ -419,6 +420,31 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			IsTruncated: false,
 			Objects:     []ObjectInfo{},
 		},
+		// ListObjectsResult-31 Empty directory, recursive listing
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+				{Name: "temporary/0/"},
+			},
+		},
+		// ListObjectsResult-32 Empty directory, non recursive listing
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+			Prefixes: []string{"temporary/"},
+		},
+		// ListObjectsResult-33 Listing empty directory only
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "temporary/0/"},
+			},
+		},
 	}
 
 	testCases := []struct {
@@ -529,67 +555,87 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		{"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "/", 10, resultCases[29], nil, true},
 		// Test with prefix and delimiter set to '/'. (60)
 		{"test-bucket-list-object", "/", "", "/", 10, resultCases[30], nil, true},
-
 		// Test with invalid prefix (61)
-		{"test-bucket-list-object", "\\", "", "/", 10, resultCases[30], ObjectNameInvalid{Bucket: "test-bucket-list-object", Object: "\\"}, false},
+		{"test-bucket-list-object", "\\", "", "/", 10, ListObjectsInfo{}, ObjectNameInvalid{Bucket: "test-bucket-list-object", Object: "\\"}, false},
+		// Test listing an empty directory in recursive mode (62)
+		{"test-bucket-empty-dir", "", "", "", 10, resultCases[31], nil, true},
+		// Test listing an empty directory in a non recursive mode (63)
+		{"test-bucket-empty-dir", "", "", "/", 10, resultCases[32], nil, true},
+		// Test listing a directory which contains an empty directory (64)
+		{"test-bucket-empty-dir", "", "temporary/", "", 10, resultCases[33], nil, true},
 	}
 
 	for i, testCase := range testCases {
-		result, err := obj.ListObjects(context.Background(), testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, int(testCase.maxKeys))
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if !strings.Contains(err.Error(), testCase.err.Error()) {
-				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error())
+		testCase := testCase
+		t.Run(fmt.Sprintf("Test%d-%s", i+1, instanceType), func(t *testing.T) {
+			result, err := obj.ListObjects(context.Background(), testCase.bucketName,
+				testCase.prefix, testCase.marker, testCase.delimeter, int(testCase.maxKeys))
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
 			}
-		}
-		// Since there are cases for which ListObjects fails, this is
-		// necessary. Test passes as expected, but the output values
-		// are verified for correctness here.
-		if err == nil && testCase.shouldPass {
-			// The length of the expected ListObjectsResult.Objects
-			// should match in both expected result from test cases
-			// and in the output. On failure calling t.Fatalf,
-			// otherwise it may lead to index out of range error in
-			// assertion following this.
-			if len(testCase.result.Objects) != len(result.Objects) {
-				t.Fatalf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.result.Objects), len(result.Objects))
+			if err == nil && !testCase.shouldPass {
+				t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error())
 			}
-			for j := 0; j < len(testCase.result.Objects); j++ {
-				if testCase.result.Objects[j].Name != result.Objects[j].Name {
-					t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name)
+			// Failed as expected, but does it fail for the expected reason.
+			if err != nil && !testCase.shouldPass {
+				if !strings.Contains(err.Error(), testCase.err.Error()) {
+					t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error())
 				}
-				//FIXME: we should always check for ETag
-				if result.Objects[j].ETag == "" && !strings.HasSuffix(result.Objects[j].Name, slashSeparator) {
-					t.Errorf("Test %d: %s: Expected ETag to be not empty, but found empty instead (%v)", i+1, instanceType, result.Objects[j].Name)
+			}
+			// Since there are cases for which ListObjects fails, this is
+			// necessary. Test passes as expected, but the output values
+			// are verified for correctness here.
+			if err == nil && testCase.shouldPass {
+				// The length of the expected ListObjectsResult.Objects
+				// should match in both expected result from test cases
+				// and in the output. On failure calling t.Fatalf,
+				// otherwise it may lead to index out of range error in
+				// assertion following this.
+				if len(testCase.result.Objects) != len(result.Objects) {
+					t.Fatalf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.result.Objects), len(result.Objects))
+				}
+				for j := 0; j < len(testCase.result.Objects); j++ {
+					if testCase.result.Objects[j].Name != result.Objects[j].Name {
+						t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name)
+					}
+					// FIXME: we should always check for ETag
+					if result.Objects[j].ETag == "" && !strings.HasSuffix(result.Objects[j].Name, slashSeparator) {
+						t.Errorf("Test %d: %s: Expected ETag to be not empty, but found empty instead (%v)", i+1, instanceType, result.Objects[j].Name)
+					}
+
+				}
+
+				if len(testCase.result.Prefixes) != len(result.Prefixes) {
+					t.Fatalf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", i+1, instanceType, len(testCase.result.Prefixes), len(result.Prefixes))
+				}
+				for j := 0; j < len(testCase.result.Prefixes); j++ {
+					if testCase.result.Prefixes[j] != result.Prefixes[j] {
+						t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Prefixes[j], result.Prefixes[j])
+					}
+				}
+
+				if testCase.result.IsTruncated != result.IsTruncated {
+					t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsTruncated, result.IsTruncated)
+				}
+
+				if testCase.result.IsTruncated && result.NextMarker == "" {
+					t.Errorf("Test %d: %s: Expected NextContinuationToken to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType)
+				}
+
+				if !testCase.result.IsTruncated && result.NextMarker != "" {
+					t.Errorf("Test %d: %s: Expected NextContinuationToken to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, result.NextMarker)
 				}
 
 			}
-			if testCase.result.IsTruncated != result.IsTruncated {
-				t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsTruncated, result.IsTruncated)
+			// Take ListObject treeWalk go-routine to completion, if available in the treewalk pool.
+			if result.IsTruncated {
+				_, err = obj.ListObjects(context.Background(), testCase.bucketName,
+					testCase.prefix, result.NextMarker, testCase.delimeter, 1000)
+				if err != nil {
+					t.Fatal(err)
+				}
 			}
-
-			if testCase.result.IsTruncated && result.NextMarker == "" {
-				t.Errorf("Test %d: %s: Expected NextContinuationToken to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType)
-			}
-
-			if !testCase.result.IsTruncated && result.NextMarker != "" {
-				t.Errorf("Test %d: %s: Expected NextContinuationToken to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, result.NextMarker)
-			}
-
-		}
-		// Take ListObject treeWalk go-routine to completion, if available in the treewalk pool.
-		if result.IsTruncated {
-			_, err = obj.ListObjects(context.Background(), testCase.bucketName, testCase.prefix, result.NextMarker, testCase.delimeter, 1000)
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
+		})
 	}
 }
 

cmd/object-api-multipart_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
 	"fmt"
 	"os"
 	"reflect"
@@ -228,8 +227,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 	}
 
 	// This causes quorum failure verify.
-	disks = disks[len(disks)-3:]
-	for _, disk := range disks {
+	for _, disk := range disks[len(disks)-3:] {
 		os.RemoveAll(disk)
 	}
 
@@ -240,9 +238,26 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
 	}
 	// as majority of xl.json are not available, we expect uploadID to be not available.
-	expectedErr := InvalidUploadID{UploadID: testCase.uploadID}
-	if err.Error() != expectedErr.Error() {
-		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr, err)
+	expectedErr1 := InsufficientReadQuorum{}
+	if err.Error() != expectedErr1.Error() {
+		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr1, err)
+	}
+
+	// This causes invalid upload id.
+	for _, disk := range disks {
+		os.RemoveAll(disk)
+	}
+
+	// Object part upload should fail with bucket not found.
+	_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
+	if err == nil {
+		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
+	}
+
+	// As all disks at not available, bucket not found.
+	expectedErr2 := BucketNotFound{Bucket: testCase.bucketName}
+	if err.Error() != expectedErr2.Error() {
+		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr2, err)
 	}
 }
 
@@ -1850,10 +1865,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 			},
 		},
 	}
-	s3MD5, err := getCompleteMultipartMD5(context.Background(), inputParts[3].parts)
-	if err != nil {
-		t.Fatalf("Obtaining S3MD5 failed")
-	}
+	s3MD5 := getCompleteMultipartMD5(inputParts[3].parts)
 
 	// Test cases with sample input values for CompleteMultipartUpload.
 	testCases := []struct {
@@ -1882,8 +1894,8 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 		// Asserting for Invalid UploadID (Test number 9).
 		{bucketNames[0], objectNames[0], "abc", []CompletePart{}, "", InvalidUploadID{UploadID: "abc"}, false},
 		// Test case with invalid Part Etag (Test number 10-11).
-		{bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abc"}}, "", hex.ErrLength, false},
-		{bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abcz"}}, "", hex.InvalidByteError(00), false},
+		{bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abc"}}, "", InvalidPart{}, false},
+		{bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abcz"}}, "", InvalidPart{}, false},
 		// Part number 0 doesn't exist, expecting InvalidPart error (Test number 12).
 		{bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abcd", PartNumber: 0}}, "", InvalidPart{}, false},
 		// // Upload and PartNumber exists, But a deliberate ETag mismatch is introduced (Test number 13).

cmd/object-api-utils.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2015, 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2015-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -190,18 +190,18 @@ func mustGetUUID() string {
 }
 
 // Create an s3 compatible MD5sum for complete multipart transaction.
-func getCompleteMultipartMD5(ctx context.Context, parts []CompletePart) (string, error) {
+func getCompleteMultipartMD5(parts []CompletePart) string {
 	var finalMD5Bytes []byte
 	for _, part := range parts {
 		md5Bytes, err := hex.DecodeString(canonicalizeETag(part.ETag))
 		if err != nil {
-			logger.LogIf(ctx, err)
-			return "", err
+			finalMD5Bytes = append(finalMD5Bytes, []byte(part.ETag)...)
+		} else {
+			finalMD5Bytes = append(finalMD5Bytes, md5Bytes...)
 		}
-		finalMD5Bytes = append(finalMD5Bytes, md5Bytes...)
 	}
 	s3MD5 := fmt.Sprintf("%s-%d", getMD5Hash(finalMD5Bytes), len(parts))
-	return s3MD5, nil
+	return s3MD5
 }
 
 // Clean unwanted fields from metadata
@@ -675,9 +675,26 @@ func (p *PutObjReader) Size() int64 {
 // as a hex encoded string
 func (p *PutObjReader) MD5CurrentHexString() string {
 	md5sumCurr := p.rawReader.MD5Current()
+	var appendHyphen bool
+	// md5sumcurr is not empty in two scenarios
+	// - server is running in strict compatibility mode
+	// - client set Content-Md5 during PUT operation
+	if len(md5sumCurr) == 0 {
+		// md5sumCurr is only empty when we are running
+		// in non-compatibility mode.
+		md5sumCurr = make([]byte, 16)
+		rand.Read(md5sumCurr)
+		appendHyphen = true
+	}
 	if p.sealMD5Fn != nil {
 		md5sumCurr = p.sealMD5Fn(md5sumCurr)
 	}
+	if appendHyphen {
+		// Make sure to return etag string upto 32 length, for SSE
+		// requests ETag might be longer and the code decrypting the
+		// ETag ignores ETag in multipart ETag form i.e <hex>-N
+		return hex.EncodeToString(md5sumCurr)[:32] + "-1"
+	}
 	return hex.EncodeToString(md5sumCurr)
 }
 

cmd/object-api-utils_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"io"
 	"net/http"
 	"reflect"
@@ -143,8 +142,8 @@ func TestGetCompleteMultipartMD5(t *testing.T) {
 		expectedResult string
 		expectedErr    string
 	}{
-		// Wrong MD5 hash string
-		{[]CompletePart{{ETag: "wrong-md5-hash-string"}}, "", "encoding/hex: invalid byte: U+0077 'w'"},
+		// Wrong MD5 hash string, returns md5um of hash
+		{[]CompletePart{{ETag: "wrong-md5-hash-string"}}, "0deb8cb07527b4b2669c861cb9653607-1", ""},
 
 		// Single CompletePart with valid MD5 hash string.
 		{[]CompletePart{{ETag: "cf1f738a5924e645913c984e0fe3d708"}}, "10dc1617fbcf0bd0858048cb96e6bd77-1", ""},
@@ -154,17 +153,10 @@ func TestGetCompleteMultipartMD5(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		result, err := getCompleteMultipartMD5(context.Background(), test.parts)
+		result := getCompleteMultipartMD5(test.parts)
 		if result != test.expectedResult {
 			t.Fatalf("test %d failed: expected: result=%v, got=%v", i+1, test.expectedResult, result)
 		}
-		errString := ""
-		if err != nil {
-			errString = err.Error()
-		}
-		if errString != test.expectedErr {
-			t.Fatalf("test %d failed: expected: err=%v, got=%v", i+1, test.expectedErr, err)
-		}
 	}
 }
 

cmd/object-handlers.go

@@ -827,7 +827,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		reader = gr
 	}
 
-	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualSize)
+	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualSize, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -934,7 +934,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 				crypto.RemoveInternalEntries(srcInfo.UserDefined)
 			}
 
-			srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", targetSize) // do not try to verify encrypted content
+			// do not try to verify encrypted content
+			srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", targetSize, globalCLIContext.StrictS3Compat)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
@@ -1176,6 +1177,11 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
+	// This request header needs to be set prior to setting ObjectOptions
+	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
+		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+	}
+
 	actualSize := size
 
 	if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
@@ -1183,7 +1189,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
 		metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
 
-		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize, globalCLIContext.StrictS3Compat)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
@@ -1196,7 +1202,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		sha256hex = ""
 	}
 
-	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -1205,10 +1211,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	rawReader := hashReader
 	pReader := NewPutObjReader(rawReader, nil, nil)
 
-	// This request header needs to be set prior to setting ObjectOptions
-	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
-		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
-	}
 	// get gateway encryption options
 	var opts ObjectOptions
 	opts, err = putOpts(ctx, r, bucket, object, metadata)
@@ -1234,7 +1236,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 				return
 			}
 			info := ObjectInfo{Size: size}
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size) // do not try to verify encrypted content
+			// do not try to verify encrypted content
+			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
@@ -1259,8 +1262,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	etag := objInfo.ETag
 	if objInfo.IsCompressed() {
-		// Ignore compressed ETag.
-		etag = objInfo.ETag + "-1"
+		if !strings.HasSuffix(objInfo.ETag, "-1") {
+			etag = objInfo.ETag + "-1"
+		}
 	} else if hasServerSideEncryptionHeader(r.Header) {
 		etag = getDecryptedETag(r.Header, objInfo, false)
 	}
@@ -1612,7 +1616,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		reader = gr
 	}
 
-	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualPartSize)
+	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualPartSize, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -1672,7 +1676,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			}
 
 			info := ObjectInfo{Size: length}
-			srcInfo.Reader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", length)
+			srcInfo.Reader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", length, globalCLIContext.StrictS3Compat)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
@@ -1839,7 +1843,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 	isCompressed := false
 	if objectAPI.IsCompressionSupported() && compressPart {
-		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize, globalCLIContext.StrictS3Compat)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
@@ -1853,7 +1857,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		isCompressed = true
 	}
 
-	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -1920,7 +1924,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 				return
 			}
 			info := ObjectInfo{Size: size}
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size) // do not try to verify encrypted content
+			// do not try to verify encrypted content
+			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
@@ -1941,10 +1946,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	}
 
 	etag := partInfo.ETag
-	if isCompressed {
-		// Suppress compressed ETag.
-		etag = partInfo.ETag + "-1"
-	} else if isEncrypted {
+	if isEncrypted {
 		etag = tryDecryptETag(objectEncryptionKey, partInfo.ETag, crypto.SSEC.IsRequested(r.Header))
 	}
 	w.Header()["ETag"] = []string{"\"" + etag + "\""}
@@ -2234,11 +2236,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	// Complete parts.
 	var completeParts []CompletePart
 	for _, part := range complMultipartUpload.Parts {
-		// Avoiding for gateway parts.
-		// `strings.TrimPrefix` does not work here as intended. So `Replace` is used instead.
-		if objectAPI.IsCompressionSupported() {
-			part.ETag = strings.Replace(part.ETag, "-1", "", -1) // For compressed multiparts, We append '-1' for part.ETag.
-		}
 		part.ETag = canonicalizeETag(part.ETag)
 		if isEncrypted {
 			// ETag is stored in the backend in encrypted form. Validate client sent ETag with

cmd/object-handlers_test.go

@@ -1916,7 +1916,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	for i, input := range putObjectInputs {
 		// uploading the object.
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData})
+		objInfo, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.md5sum, ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -2233,11 +2233,6 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 				t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i+1, instanceType, err)
 			}
 
-			retag := canonicalizeETag(cpObjResp.ETag)
-			if retag != bytesData[0].md5sum {
-				t.Fatalf("Test %d: %s: Failed to CopyObject: <ERROR> got %s, expected %s", i+1, instanceType, retag, bytesData[0].md5sum)
-			}
-
 			// See if the new object is formed.
 			// testing whether the copy was successful.
 			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
@@ -2660,10 +2655,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 	}
 
 	// on successful complete multipart operation the s3MD5 for the parts uploaded will be returned.
-	s3MD5, err := getCompleteMultipartMD5(context.Background(), inputParts[3].parts)
-	if err != nil {
-		t.Fatalf("Obtaining S3MD5 failed")
-	}
+	s3MD5 := getCompleteMultipartMD5(inputParts[3].parts)
 
 	// generating the response body content for the success case.
 	successResponse := generateCompleteMultpartUploadResponse(bucketName, objectName, getGetObjectURL("", bucketName, objectName), s3MD5)

cmd/posix-errors.go

@@ -136,3 +136,12 @@ func isSysErrCrossDevice(err error) bool {
 	e, ok := err.(*os.LinkError)
 	return ok && e.Err == syscall.EXDEV
 }
+
+// Check if given error corresponds to too many open files
+func isSysErrTooManyFiles(err error) bool {
+	if err == syscall.ENFILE || err == syscall.EMFILE {
+		return true
+	}
+	pathErr, ok := err.(*os.PathError)
+	return ok && (pathErr.Err == syscall.ENFILE || pathErr.Err == syscall.EMFILE)
+}

cmd/posix-list-dir_unix.go

@@ -20,7 +20,6 @@ package cmd
 
 import (
 	"os"
-	"path"
 	"runtime"
 	"sync"
 	"syscall"
@@ -79,7 +78,7 @@ func parseDirents(dirPath string, buf []byte) (entries []string, err error) {
 			// On Linux XFS does not implement d_type for on disk
 			// format << v5. Fall back to OsStat().
 			var fi os.FileInfo
-			fi, err = os.Stat(path.Join(dirPath, name))
+			fi, err = os.Stat(pathJoin(dirPath, name))
 			if err != nil {
 				// If file does not exist, we continue and skip it.
 				// Could happen if it was deleted in the middle while

cmd/posix-list-dir_windows.go

@@ -20,7 +20,6 @@ package cmd
 
 import (
 	"os"
-	"path"
 	"strings"
 	"syscall"
 )
@@ -82,7 +81,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 		case data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0:
 			// If its symbolic link, follow the link using os.Stat()
 			var fi os.FileInfo
-			fi, err = os.Stat(path.Join(dirPath, name))
+			fi, err = os.Stat(pathJoin(dirPath, name))
 			if err != nil {
 				// If file does not exist, we continue and skip it.
 				// Could happen if it was deleted in the middle while

cmd/posix.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,6 +25,7 @@ import (
 	slashpath "path"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -34,15 +35,19 @@ import (
 	"bytes"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/klauspost/readahead"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
+	xioutil "github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/mountinfo"
+	"github.com/ncw/directio"
 )
 
 const (
 	diskMinFreeSpace  = 900 * humanize.MiByte // Min 900MiB free space.
 	diskMinTotalSpace = diskMinFreeSpace      // Min 900MiB total space.
 	maxAllowedIOError = 5
+	readBlockSize     = humanize.KiByte * 32 // Default read block size 32KiB.
 )
 
 // isValidVolname verifies a volname name in accordance with object
@@ -71,7 +76,6 @@ type posix struct {
 	connected bool
 
 	diskMount bool // indicates if the path is an actual mount.
-	driveSync bool // indicates if the backend is synchronous.
 
 	diskFileInfo os.FileInfo
 	// Disk usage metrics
@@ -185,10 +189,9 @@ func newPosix(path string) (*posix, error) {
 	p := &posix{
 		connected: true,
 		diskPath:  path,
-		// 1MiB buffer pool for posix internal operations.
 		pool: sync.Pool{
 			New: func() interface{} {
-				b := make([]byte, readSizeV1)
+				b := directio.AlignedBlock(readBlockSize)
 				return &b
 			},
 		},
@@ -197,15 +200,6 @@ func newPosix(path string) (*posix, error) {
 		diskMount:    mountinfo.IsLikelyMountPoint(path),
 	}
 
-	var pf BoolFlag
-	if driveSync := os.Getenv("MINIO_DRIVE_SYNC"); driveSync != "" {
-		pf, err = ParseBoolFlag(driveSync)
-		if err != nil {
-			return nil, err
-		}
-		p.driveSync = bool(pf)
-	}
-
 	if !p.diskMount {
 		go p.diskUsage(GlobalServiceDoneCh)
 	}
@@ -649,9 +643,88 @@ func (s *posix) DeleteVol(volume string) (err error) {
 	return nil
 }
 
+// Walk - is a sorted walker which returns file entries in lexically
+// sorted order, additionally along with metadata about each of those entries.
+func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile string,
+	readMetadataFn readMetadataFunc, endWalkCh chan struct{}) (ch chan FileInfo, err error) {
+
+	defer func() {
+		if err == errFaultyDisk {
+			atomic.AddInt32(&s.ioErrCount, 1)
+		}
+	}()
+
+	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
+		return nil, errFaultyDisk
+	}
+
+	if err = s.checkDiskFound(); err != nil {
+		return nil, err
+	}
+
+	// Verify if volume is valid and it exists.
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return nil, err
+	}
+
+	// Stat a volume entry.
+	_, err = os.Stat(volumeDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
+		}
+		return nil, err
+	}
+
+	ch = make(chan FileInfo)
+	go func() {
+		defer close(ch)
+		listDir := func(volume, dirPath, dirEntry string) (entries []string) {
+			entries, err := s.ListDir(volume, dirPath, -1, leafFile)
+			if err != nil {
+				return
+			}
+			sort.Strings(entries)
+			return filterMatchingPrefix(entries, dirEntry)
+		}
+
+		walkResultCh := startTreeWalk(context.Background(), volume, dirPath, marker, recursive, listDir, endWalkCh)
+		for {
+			walkResult, ok := <-walkResultCh
+			if !ok {
+				return
+			}
+			var fi FileInfo
+			if hasSuffix(walkResult.entry, slashSeparator) {
+				fi = FileInfo{
+					Volume: volume,
+					Name:   walkResult.entry,
+					Mode:   os.ModeDir,
+				}
+			} else {
+				buf, err := s.ReadAll(volume, pathJoin(walkResult.entry, leafFile))
+				if err != nil {
+					continue
+				}
+				fi = readMetadataFn(buf, volume, walkResult.entry)
+			}
+			select {
+			case ch <- fi:
+			case <-endWalkCh:
+				return
+			}
+		}
+	}()
+
+	return ch, nil
+}
+
 // ListDir - return all the entries at the given directory path.
 // If an entry is a directory it will be returned with a trailing "/".
-func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, err error) {
+func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (entries []string, err error) {
 	defer func() {
 		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
@@ -684,9 +757,21 @@ func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, er
 
 	dirPath = pathJoin(volumeDir, dirPath)
 	if count > 0 {
-		return readDirN(dirPath, count)
+		entries, err = readDirN(dirPath, count)
+	} else {
+		entries, err = readDir(dirPath)
 	}
-	return readDir(dirPath)
+
+	// If leaf file is specified, filter out the entries.
+	if leafFile != "" {
+		for i, entry := range entries {
+			if _, serr := os.Stat(pathJoin(dirPath, entry, leafFile)); serr == nil {
+				entries[i] = strings.TrimSuffix(entry, slashSeparator)
+			}
+		}
+	}
+
+	return entries, err
 }
 
 // ReadAll reads from r until an error or EOF and returns the data it read.
@@ -721,6 +806,8 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 			return nil, errVolumeNotFound
 		} else if isSysErrIO(err) {
 			return nil, errFaultyDisk
+		} else if isSysErrTooManyFiles(err) {
+			return nil, errTooManyOpenFiles
 		}
 		return nil, err
 	}
@@ -824,6 +911,8 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 			return 0, errFileAccessDenied
 		case isSysErrIO(err):
 			return 0, errFaultyDisk
+		case isSysErrTooManyFiles(err):
+			return 0, errTooManyOpenFiles
 		default:
 			return 0, err
 		}
@@ -934,6 +1023,8 @@ func (s *posix) openFile(volume, path string, mode int) (f *os.File, err error)
 			return nil, errFileAccessDenied
 		case isSysErrIO(err):
 			return nil, errFaultyDisk
+		case isSysErrTooManyFiles(err):
+			return nil, errTooManyOpenFiles
 		default:
 			return nil, err
 		}
@@ -996,6 +1087,8 @@ func (s *posix) ReadFileStream(volume, path string, offset, length int64) (io.Re
 			return nil, errFileAccessDenied
 		case isSysErrIO(err):
 			return nil, errFaultyDisk
+		case isSysErrTooManyFiles(err):
+			return nil, errTooManyOpenFiles
 		default:
 			return nil, err
 		}
@@ -1015,18 +1108,20 @@ func (s *posix) ReadFileStream(volume, path string, offset, length int64) (io.Re
 	if _, err = file.Seek(offset, io.SeekStart); err != nil {
 		return nil, err
 	}
-	return struct {
+
+	r := struct {
 		io.Reader
 		io.Closer
-	}{Reader: io.LimitReader(file, length), Closer: file}, nil
+	}{Reader: io.LimitReader(file, length), Closer: file}
+
+	return readahead.NewReadCloser(r), nil
 }
 
 // CreateFile - creates the file.
 func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (err error) {
-	if fileSize < 0 {
+	if fileSize < -1 {
 		return errInvalidArgument
 	}
-
 	defer func() {
 		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
@@ -1045,14 +1140,49 @@ func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (er
 		return err
 	}
 
-	// Create file if not found. Note that it is created with os.O_EXCL flag as the file
-	// always is supposed to be created in the tmp directory with a unique file name.
-	w, err := s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY|os.O_EXCL)
-	if err != nil {
+	if err = s.checkDiskFound(); err != nil {
 		return err
 	}
 
-	defer w.Close()
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return err
+	}
+	// Stat a volume entry.
+	_, err = os.Stat((volumeDir))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
+		}
+		return err
+	}
+
+	filePath := pathJoin(volumeDir, path)
+	if err = checkPathLength((filePath)); err != nil {
+		return err
+	}
+
+	// Create top level directories if they don't exist.
+	// with mode 0777 mkdir honors system umask.
+	if err = mkdirAll(slashpath.Dir(filePath), 0777); err != nil {
+		return err
+	}
+
+	w, err := disk.OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0666)
+	if err != nil {
+		switch {
+		case os.IsPermission(err):
+			return errFileAccessDenied
+		case os.IsExist(err):
+			return errFileAccessDenied
+		case isSysErrIO(err):
+			return errFaultyDisk
+		default:
+			return err
+		}
+	}
 
 	var e error
 	if fileSize > 0 {
@@ -1075,23 +1205,26 @@ func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (er
 		return err
 	}
 
+	defer w.Close()
+
 	bufp := s.pool.Get().(*[]byte)
 	defer s.pool.Put(bufp)
 
-	n, err := io.CopyBuffer(w, r, *bufp)
+	written, err := xioutil.CopyAligned(w, r, *bufp)
 	if err != nil {
 		return err
 	}
-	if n < fileSize {
+
+	if written < fileSize {
 		return errLessData
-	}
-	if n > fileSize {
+	} else if written > fileSize {
 		return errMoreData
 	}
+
 	return nil
 }
 
-func (s *posix) WriteAll(volume, path string, buf []byte) (err error) {
+func (s *posix) WriteAll(volume, path string, reader io.Reader) (err error) {
 	defer func() {
 		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
@@ -1109,11 +1242,13 @@ func (s *posix) WriteAll(volume, path string, buf []byte) (err error) {
 		return err
 	}
 
-	if _, err = w.Write(buf); err != nil {
-		return err
-	}
+	defer w.Close()
 
-	return w.Close()
+	bufp := s.pool.Get().(*[]byte)
+	defer s.pool.Put(bufp)
+
+	_, err = io.CopyBuffer(w, reader, *bufp)
+	return err
 }
 
 // AppendFile - append a byte array at path, if file doesn't exist at
@@ -1130,13 +1265,9 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 	}
 
 	var w *os.File
-	// Create file if not found, additionally also enables synchronous
-	// operation if asked by the user.
-	if s.driveSync {
-		w, err = s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_APPEND|os.O_WRONLY)
-	} else {
-		w, err = s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY)
-	}
+	// Create file if not found. Not doing O_DIRECT here to avoid the code that does buffer aligned writes.
+	// AppendFile() is only used by healing code to heal objects written in old format.
+	w, err = s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_APPEND|os.O_WRONLY)
 	if err != nil {
 		return err
 	}
@@ -1290,6 +1421,14 @@ func (s *posix) DeleteFile(volume, path string) (err error) {
 	return deleteFile(volumeDir, filePath)
 }
 
+func (s *posix) DeleteFileBulk(volume string, paths []string) (errs []error, err error) {
+	errs = make([]error, len(paths))
+	for idx, path := range paths {
+		errs[idx] = s.DeleteFile(volume, path)
+	}
+	return
+}
+
 // RenameFile - rename source path to destination path atomically.
 func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
 	defer func() {

cmd/posix_test.go

@@ -839,7 +839,7 @@ func TestPosixPosixListDir(t *testing.T) {
 		} else {
 			t.Errorf("Expected the StorageAPI to be of type *posix")
 		}
-		dirList, err = posixStorage.ListDir(testCase.srcVol, testCase.srcPath, -1)
+		dirList, err = posixStorage.ListDir(testCase.srcVol, testCase.srcPath, -1, "")
 		if err != testCase.expectedErr {
 			t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}

cmd/rest/client.go

@@ -89,12 +89,7 @@ func newCustomDialContext(timeout time.Duration) func(ctx context.Context, netwo
 			DualStack: true,
 		}
 
-		conn, err := dialer.DialContext(ctx, network, addr)
-		if err != nil {
-			return nil, err
-		}
-
-		return xhttp.NewTimeoutConn(conn, timeout, timeout), nil
+		return dialer.DialContext(ctx, network, addr)
 	}
 }
 
@@ -105,9 +100,8 @@ func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAu
 	tr := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		DialContext:           newCustomDialContext(timeout),
-		MaxIdleConnsPerHost:   4096,
-		MaxIdleConns:          4096,
-		IdleConnTimeout:       120 * time.Second,
+		MaxIdleConnsPerHost:   256,
+		IdleConnTimeout:       60 * time.Second,
 		TLSHandshakeTimeout:   30 * time.Second,
 		ExpectContinueTimeout: 10 * time.Second,
 		TLSClientConfig:       tlsConfig,

cmd/routers.go

@@ -42,7 +42,7 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 	registerPeerRESTHandlers(router)
 
 	// Register distributed namespace lock.
-	registerDistNSLockRouter(router)
+	registerLockRESTHandlers(router)
 
 }
 
@@ -54,8 +54,6 @@ var globalHandlers = []HandlerFunc{
 	addSecurityHeaders,
 	// Forward path style requests to actual host in a bucket federated setup.
 	setBucketForwardingHandler,
-	// Ratelimit the incoming requests using a token bucket algorithm
-	setRateLimitHandler,
 	// Validate all the incoming requests.
 	setRequestValidityHandler,
 	// Network statistics

cmd/rpc.go

@@ -1,281 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/tls"
-	"fmt"
-	"net"
-	"net/url"
-	"sync"
-	"time"
-
-	xrpc "github.com/minio/minio/cmd/rpc"
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-// DefaultSkewTime - skew time is 15 minutes between minio peers.
-const DefaultSkewTime = 15 * time.Minute
-
-// defaultRPCTimeout - default RPC timeout is one minute.
-const defaultRPCTimeout = 5 * time.Minute
-
-// defaultRPCRetryTime - default RPC time to wait before retry after a network error
-const defaultRPCRetryTime = 1 * time.Minute
-
-var errRPCRetry = fmt.Errorf("rpc: retry error")
-
-func isNetError(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	if uerr, isURLError := err.(*url.Error); isURLError {
-		if uerr.Timeout() {
-			return true
-		}
-
-		err = uerr.Err
-	}
-
-	_, isNetOpError := err.(*net.OpError)
-	return isNetOpError
-}
-
-// RPCVersion - RPC semantic version based on semver 2.0.0 https://semver.org/.
-type RPCVersion struct {
-	Major uint64
-	Minor uint64
-	Patch uint64
-}
-
-// Compare - compares given version with this version.
-func (v RPCVersion) Compare(o RPCVersion) int {
-	compare := func(v1, v2 uint64) int {
-		if v1 == v2 {
-			return 0
-		}
-
-		if v1 > v2 {
-			return 1
-		}
-
-		return -1
-	}
-
-	if r := compare(v.Major, o.Major); r != 0 {
-		return r
-	}
-
-	if r := compare(v.Minor, o.Minor); r != 0 {
-		return r
-	}
-
-	return compare(v.Patch, o.Patch)
-}
-
-func (v RPCVersion) String() string {
-	return fmt.Sprintf("%v.%v.%v", v.Major, v.Minor, v.Patch)
-}
-
-// AuthArgs - base argument for any RPC call for authentication.
-type AuthArgs struct {
-	Token       string
-	RPCVersion  RPCVersion
-	RequestTime time.Time
-}
-
-// Authenticate - checks if given arguments are valid to allow RPC call.
-// This is xrpc.Authenticator and is called in RPC server.
-func (args AuthArgs) Authenticate() error {
-	// Check whether request time is within acceptable skew time.
-	utcNow := time.Now().UTC()
-	if args.RequestTime.Sub(utcNow) > DefaultSkewTime || utcNow.Sub(args.RequestTime) > DefaultSkewTime {
-		return fmt.Errorf("client time %v is too apart with server time %v", args.RequestTime, utcNow)
-	}
-
-	if globalRPCAPIVersion.Compare(args.RPCVersion) != 0 {
-		return fmt.Errorf("version mismatch. expected: %v, received: %v", globalRPCAPIVersion, args.RPCVersion)
-	}
-
-	if !isAuthTokenValid(args.Token) {
-		return errAuthentication
-	}
-
-	return nil
-}
-
-// SetAuthArgs - sets given authentication arguments to this args. This is called in RPC client.
-func (args *AuthArgs) SetAuthArgs(authArgs AuthArgs) {
-	*args = authArgs
-}
-
-// VoidReply - void (empty) RPC reply.
-type VoidReply struct{}
-
-// RPCClientArgs - RPC client arguments.
-type RPCClientArgs struct {
-	NewAuthTokenFunc func() string
-	RPCVersion       RPCVersion
-	ServiceName      string
-	ServiceURL       *xnet.URL
-	TLSConfig        *tls.Config
-}
-
-// validate - checks whether given args are valid or not.
-func (args RPCClientArgs) validate() error {
-	if args.NewAuthTokenFunc == nil {
-		return fmt.Errorf("NewAuthTokenFunc must not be empty")
-	}
-
-	if args.ServiceName == "" {
-		return fmt.Errorf("ServiceName must not be empty")
-	}
-
-	if args.ServiceURL.Scheme != "http" && args.ServiceURL.Scheme != "https" {
-		return fmt.Errorf("unknown RPC URL %v", args.ServiceURL)
-	}
-
-	if args.ServiceURL.User != nil || args.ServiceURL.ForceQuery || args.ServiceURL.RawQuery != "" || args.ServiceURL.Fragment != "" {
-		return fmt.Errorf("unknown RPC URL %v", args.ServiceURL)
-	}
-
-	if args.ServiceURL.Scheme == "https" && args.TLSConfig == nil {
-		return fmt.Errorf("tls configuration must not be empty for https url %v", args.ServiceURL)
-	}
-
-	return nil
-}
-
-// RPCClient - base RPC client.
-type RPCClient struct {
-	sync.RWMutex
-	args        RPCClientArgs
-	authToken   string
-	rpcClient   *xrpc.Client
-	retryTicker *time.Ticker
-}
-
-func (client *RPCClient) setRetryTicker(ticker *time.Ticker) {
-	if ticker == nil {
-		client.RLock()
-		isNil := client.retryTicker == nil
-		client.RUnlock()
-		if isNil {
-			return
-		}
-	}
-
-	client.Lock()
-	defer client.Unlock()
-
-	if client.retryTicker != nil {
-		client.retryTicker.Stop()
-	}
-
-	client.retryTicker = ticker
-}
-
-// Call - calls servicemethod on remote server.
-func (client *RPCClient) Call(serviceMethod string, args interface {
-	SetAuthArgs(args AuthArgs)
-}, reply interface{}) (err error) {
-	lockedCall := func() error {
-		client.RLock()
-		retryTicker := client.retryTicker
-		client.RUnlock()
-		if retryTicker != nil {
-			select {
-			case <-retryTicker.C:
-			default:
-				return errRPCRetry
-			}
-		}
-
-		client.RLock()
-		authToken := client.authToken
-		client.RUnlock()
-
-		// Make RPC call.
-		args.SetAuthArgs(AuthArgs{authToken, client.args.RPCVersion, time.Now().UTC()})
-		return client.rpcClient.Call(serviceMethod, args, reply)
-	}
-
-	call := func() error {
-		err = lockedCall()
-
-		if err == errRPCRetry {
-			return err
-		}
-
-		if isNetError(err) {
-			client.setRetryTicker(time.NewTicker(defaultRPCRetryTime))
-		} else {
-			client.setRetryTicker(nil)
-		}
-
-		return err
-	}
-
-	// If authentication error is received, retry the same call only once
-	// with new authentication token.
-	if err = call(); err == nil {
-		return nil
-	}
-	if err.Error() != errAuthentication.Error() {
-		return err
-	}
-
-	client.Lock()
-	client.authToken = client.args.NewAuthTokenFunc()
-	client.Unlock()
-	return call()
-}
-
-// Close - closes underneath RPC client.
-func (client *RPCClient) Close() error {
-	client.Lock()
-	defer client.Unlock()
-
-	client.authToken = ""
-	return client.rpcClient.Close()
-}
-
-// ServiceURL - returns service URL used for RPC call.
-func (client *RPCClient) ServiceURL() *xnet.URL {
-	// Take copy of ServiceURL
-	u := *(client.args.ServiceURL)
-	return &u
-}
-
-// NewRPCClient - returns new RPC client.
-func NewRPCClient(args RPCClientArgs) (*RPCClient, error) {
-	if err := args.validate(); err != nil {
-		return nil, err
-	}
-
-	rpcClient, err := xrpc.NewClient(args.ServiceURL, args.TLSConfig, defaultRPCTimeout)
-	if err != nil {
-		return nil, err
-	}
-
-	return &RPCClient{
-		args:      args,
-		authToken: args.NewAuthTokenFunc(),
-		rpcClient: rpcClient,
-	}, nil
-}

cmd/rpc/client.go

@@ -1,142 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rpc
-
-import (
-	"bytes"
-	"context"
-	"crypto/tls"
-	"encoding/gob"
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-	"reflect"
-	"time"
-
-	xhttp "github.com/minio/minio/cmd/http"
-	xnet "github.com/minio/minio/pkg/net"
-	"golang.org/x/net/http2"
-)
-
-// DefaultRPCTimeout - default RPC timeout is one minute.
-const DefaultRPCTimeout = 1 * time.Minute
-
-// Client - http based RPC client.
-type Client struct {
-	httpClient          *http.Client
-	httpIdleConnsCloser func()
-	serviceURL          *xnet.URL
-}
-
-// Call - calls service method on RPC server.
-func (client *Client) Call(serviceMethod string, args, reply interface{}) error {
-	replyKind := reflect.TypeOf(reply).Kind()
-	if replyKind != reflect.Ptr {
-		return fmt.Errorf("rpc reply must be a pointer type, but found %v", replyKind)
-	}
-
-	argBuf := bytes.NewBuffer(make([]byte, 0, 1024))
-
-	if err := gobEncodeBuf(args, argBuf); err != nil {
-		return err
-	}
-
-	callRequest := CallRequest{
-		Method:   serviceMethod,
-		ArgBytes: argBuf.Bytes(),
-	}
-
-	reqBuf := bytes.NewBuffer(make([]byte, 0, 1024))
-	if err := gob.NewEncoder(reqBuf).Encode(callRequest); err != nil {
-		return err
-	}
-
-	response, err := client.httpClient.Post(client.serviceURL.String(), "", reqBuf)
-	if err != nil {
-		return err
-	}
-	defer xhttp.DrainBody(response.Body)
-
-	if response.StatusCode != http.StatusOK {
-		return fmt.Errorf("%v rpc call failed with error code %v", serviceMethod, response.StatusCode)
-	}
-
-	var callResponse CallResponse
-	if err := gob.NewDecoder(response.Body).Decode(&callResponse); err != nil {
-		return err
-	}
-
-	if callResponse.Error != "" {
-		return errors.New(callResponse.Error)
-	}
-
-	return gobDecode(callResponse.ReplyBytes, reply)
-}
-
-// Close closes all idle connections of the underlying http client
-func (client *Client) Close() error {
-	if client.httpIdleConnsCloser != nil {
-		client.httpIdleConnsCloser()
-	}
-	return nil
-}
-
-func newCustomDialContext(timeout time.Duration) func(ctx context.Context, network, addr string) (net.Conn, error) {
-	return func(ctx context.Context, network, addr string) (net.Conn, error) {
-		dialer := &net.Dialer{
-			Timeout:   timeout,
-			KeepAlive: timeout,
-			DualStack: true,
-		}
-
-		conn, err := dialer.DialContext(ctx, network, addr)
-		if err != nil {
-			return nil, err
-		}
-
-		return xhttp.NewTimeoutConn(conn, timeout, timeout), nil
-	}
-}
-
-// NewClient - returns new RPC client.
-func NewClient(serviceURL *xnet.URL, tlsConfig *tls.Config, timeout time.Duration) (*Client, error) {
-	// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
-	// except custom DialContext and TLSClientConfig.
-	tr := &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           newCustomDialContext(timeout),
-		MaxIdleConnsPerHost:   4096,
-		MaxIdleConns:          4096,
-		IdleConnTimeout:       120 * time.Second,
-		TLSHandshakeTimeout:   30 * time.Second,
-		ExpectContinueTimeout: 10 * time.Second,
-		TLSClientConfig:       tlsConfig,
-		DisableCompression:    true,
-	}
-	if tlsConfig != nil {
-		// If TLS is enabled configure http2
-		if err := http2.ConfigureTransport(tr); err != nil {
-			return nil, err
-		}
-	}
-	return &Client{
-		httpClient:          &http.Client{Transport: tr},
-		httpIdleConnsCloser: tr.CloseIdleConnections,
-		serviceURL:          serviceURL,
-	}, nil
-}

cmd/rpc/client_test.go

@@ -1,75 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rpc
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-func TestClientCall(t *testing.T) {
-	rpcServer := NewServer()
-	if err := rpcServer.RegisterName("Arith", &Arith{}); err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		rpcServer.ServeHTTP(w, r)
-	}))
-	defer httpServer.Close()
-
-	url, err := xnet.ParseURL(httpServer.URL)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	rpcClient, err := NewClient(url, nil, DefaultRPCTimeout)
-	if err != nil {
-		t.Fatalf("NewClient initialization error %v", err)
-	}
-
-	var reply int
-	var boolReply bool
-	var intArg int
-
-	testCases := []struct {
-		serviceMethod string
-		args          interface{}
-		reply         interface{}
-		expectErr     bool
-	}{
-		{"Arith.Multiply", Args{7, 8}, &reply, false},
-		{"Arith.Multiply", &Args{7, 8}, &reply, false},
-		// rpc reply must be a pointer type but found int error.
-		{"Arith.Multiply", &Args{7, 8}, reply, true},
-		// gob: type mismatch in decoder: want struct type rpc.Args; got non-struct error.
-		{"Arith.Multiply", intArg, &reply, true},
-		// gob: decoding into local type *bool, received remote type int error.
-		{"Arith.Multiply", &Args{7, 8}, &boolReply, true},
-	}
-
-	for i, testCase := range testCases {
-		err := rpcClient.Call(testCase.serviceMethod, testCase.args, testCase.reply)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}

cmd/rpc/server.go

@@ -1,256 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rpc
-
-import (
-	"bytes"
-	"encoding/gob"
-	"errors"
-	"fmt"
-	"net/http"
-	"reflect"
-	"strings"
-	"unicode"
-	"unicode/utf8"
-)
-
-// Authenticator - validator of first argument of any RPC call.
-type Authenticator interface {
-	// Method to validate first argument of any RPC call.
-	Authenticate() error
-}
-
-// reflect.Type of error interface.
-var errorType = reflect.TypeOf((*error)(nil)).Elem()
-
-// reflect.Type of Authenticator interface.
-var authenticatorType = reflect.TypeOf((*Authenticator)(nil)).Elem()
-
-func gobEncodeBuf(e interface{}, buf *bytes.Buffer) error {
-	return gob.NewEncoder(buf).Encode(e)
-}
-
-func gobDecode(data []byte, e interface{}) error {
-	return gob.NewDecoder(bytes.NewReader(data)).Decode(e)
-}
-
-// Returns whether given type is exported or builin type or not.
-func isExportedOrBuiltinType(t reflect.Type) bool {
-	for t.Kind() == reflect.Ptr {
-		t = t.Elem()
-	}
-
-	rune, _ := utf8.DecodeRuneInString(t.Name())
-	return unicode.IsUpper(rune) || t.PkgPath() == ""
-}
-
-// Makes method name map from given type.
-func getMethodMap(receiverType reflect.Type) map[string]reflect.Method {
-	methodMap := make(map[string]reflect.Method)
-	for i := 0; i < receiverType.NumMethod(); i++ {
-		// Method.PkgPath is empty for this package.
-		method := receiverType.Method(i)
-
-		// Methods must have three arguments (receiver, args, reply)
-		if method.Type.NumIn() != 3 {
-			continue
-		}
-
-		// First argument must be exported.
-		if !isExportedOrBuiltinType(method.Type.In(1)) {
-			continue
-		}
-
-		// First argument must be Authenticator.
-		if !method.Type.In(1).Implements(authenticatorType) {
-			continue
-		}
-
-		// Second argument must be exported or builtin type.
-		if !isExportedOrBuiltinType(method.Type.In(2)) {
-			continue
-		}
-
-		// Second argument must be a pointer.
-		if method.Type.In(2).Kind() != reflect.Ptr {
-			continue
-		}
-
-		// Method must return one value.
-		if method.Type.NumOut() != 1 {
-			continue
-		}
-
-		// The return type of the method must be error.
-		if method.Type.Out(0) != errorType {
-			continue
-		}
-
-		methodMap[method.Name] = method
-	}
-
-	return methodMap
-}
-
-// Server - HTTP based RPC server.
-type Server struct {
-	serviceName   string
-	receiverValue reflect.Value
-	methodMap     map[string]reflect.Method
-}
-
-// RegisterName - registers receiver with given name to handle RPC requests.
-func (server *Server) RegisterName(name string, receiver interface{}) error {
-	server.serviceName = name
-
-	server.receiverValue = reflect.ValueOf(receiver)
-	if !reflect.Indirect(server.receiverValue).IsValid() {
-		return fmt.Errorf("nil receiver")
-	}
-
-	receiverName := reflect.Indirect(server.receiverValue).Type().Name()
-	receiverType := reflect.TypeOf(receiver)
-	server.methodMap = getMethodMap(receiverType)
-	if len(server.methodMap) == 0 {
-		str := "rpc.Register: type " + receiverName + " has no exported methods of suitable type"
-
-		// To help the user, see if a pointer receiver would work.
-		if len(getMethodMap(reflect.PtrTo(receiverType))) != 0 {
-			str += " (hint: pass a pointer to value of that type)"
-		}
-
-		return errors.New(str)
-	}
-
-	return nil
-}
-
-// call - call service method in receiver.
-func (server *Server) call(serviceMethod string, argBytes []byte, replyBytes *bytes.Buffer) (err error) {
-	tokens := strings.SplitN(serviceMethod, ".", 2)
-	if len(tokens) != 2 {
-		return fmt.Errorf("invalid service/method request ill-formed %v", serviceMethod)
-	}
-
-	serviceName := tokens[0]
-	if serviceName != server.serviceName {
-		return fmt.Errorf("can't find service %v", serviceName)
-	}
-
-	methodName := tokens[1]
-	method, found := server.methodMap[methodName]
-	if !found {
-		return fmt.Errorf("can't find method %v", methodName)
-	}
-
-	var argv reflect.Value
-
-	// Decode the argument value.
-	argIsValue := false // if true, need to indirect before calling.
-	if method.Type.In(1).Kind() == reflect.Ptr {
-		argv = reflect.New(method.Type.In(1).Elem())
-	} else {
-		argv = reflect.New(method.Type.In(1))
-		argIsValue = true
-	}
-
-	if err = gobDecode(argBytes, argv.Interface()); err != nil {
-		return err
-	}
-
-	if argIsValue {
-		argv = argv.Elem()
-	}
-
-	// call Authenticate() method.
-	authMethod, ok := method.Type.In(1).MethodByName("Authenticate")
-	if !ok {
-		panic("Authenticate() method not found. This should not happen.")
-	}
-	returnValues := authMethod.Func.Call([]reflect.Value{argv})
-	errInter := returnValues[0].Interface()
-	if errInter != nil {
-		err = errInter.(error)
-	}
-	if err != nil {
-		return err
-	}
-
-	replyv := reflect.New(method.Type.In(2).Elem())
-
-	switch method.Type.In(2).Elem().Kind() {
-	case reflect.Map:
-		replyv.Elem().Set(reflect.MakeMap(method.Type.In(2).Elem()))
-	case reflect.Slice:
-		replyv.Elem().Set(reflect.MakeSlice(method.Type.In(2).Elem(), 0, 0))
-	}
-
-	returnValues = method.Func.Call([]reflect.Value{server.receiverValue, argv, replyv})
-	errInter = returnValues[0].Interface()
-	if errInter != nil {
-		err = errInter.(error)
-	}
-	if err != nil {
-		return err
-	}
-
-	return gobEncodeBuf(replyv.Interface(), replyBytes)
-}
-
-// CallRequest - RPC call request parameters.
-type CallRequest struct {
-	Method   string
-	ArgBytes []byte
-}
-
-// CallResponse - RPC call response parameters.
-type CallResponse struct {
-	Error      string
-	ReplyBytes []byte
-}
-
-// ServeHTTP - handles RPC on HTTP request.
-func (server *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
-	if req.Method != http.MethodPost {
-		w.WriteHeader(http.StatusMethodNotAllowed)
-		return
-	}
-
-	var callRequest CallRequest
-	if err := gob.NewDecoder(req.Body).Decode(&callRequest); err != nil {
-		w.WriteHeader(http.StatusBadRequest)
-		return
-	}
-
-	callResponse := CallResponse{}
-
-	buf := bytes.NewBuffer(make([]byte, 0, 1024))
-
-	if err := server.call(callRequest.Method, callRequest.ArgBytes, buf); err != nil {
-		callResponse.Error = err.Error()
-	}
-	callResponse.ReplyBytes = buf.Bytes()
-
-	gob.NewEncoder(w).Encode(callResponse)
-
-	w.(http.Flusher).Flush()
-}
-
-// NewServer - returns new RPC server.
-func NewServer() *Server {
-	return &Server{}
-}

cmd/rpc/server_test.go

@@ -1,349 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rpc
-
-import (
-	"bytes"
-	"encoding/gob"
-	"errors"
-	"net/http"
-	"net/http/httptest"
-	"reflect"
-	"testing"
-)
-
-func gobEncode(e interface{}) ([]byte, error) {
-	var buf bytes.Buffer
-	err := gob.NewEncoder(&buf).Encode(e)
-	return buf.Bytes(), err
-}
-
-type Args struct {
-	A, B int
-}
-
-func (a *Args) Authenticate() (err error) {
-	if a.A == 0 && a.B == 0 {
-		err = errors.New("authenticated failed")
-	}
-
-	return
-}
-
-type Quotient struct {
-	Quo, Rem int
-}
-
-type Arith struct{}
-
-func (t *Arith) Multiply(args *Args, reply *int) error {
-	*reply = args.A * args.B
-	return nil
-}
-
-func (t *Arith) Divide(args *Args, quo *Quotient) error {
-	if args.B == 0 {
-		return errors.New("divide by zero")
-	}
-	quo.Quo = args.A / args.B
-	quo.Rem = args.A % args.B
-	return nil
-}
-
-type mytype int
-
-type Auth struct{}
-
-func (a Auth) Authenticate() error {
-	return nil
-}
-
-// exported method.
-func (t mytype) Foo(a *Auth, b *int) error {
-	return nil
-}
-
-// incompatible method because of first argument is not Authenticator.
-func (t *mytype) Bar(a, b *int) error {
-	return nil
-}
-
-// incompatible method because of error is not returned.
-func (t mytype) IncompatFoo(a, b *int) {
-}
-
-// incompatible method because of second argument is not a pointer.
-func (t *mytype) IncompatBar(a *int, b int) error {
-	return nil
-}
-
-func TestIsExportedOrBuiltinType(t *testing.T) {
-	var i int
-	case1Type := reflect.TypeOf(i)
-
-	var iptr *int
-	case2Type := reflect.TypeOf(iptr)
-
-	var a Arith
-	case3Type := reflect.TypeOf(a)
-
-	var aptr *Arith
-	case4Type := reflect.TypeOf(aptr)
-
-	var m mytype
-	case5Type := reflect.TypeOf(m)
-
-	var mptr *mytype
-	case6Type := reflect.TypeOf(mptr)
-
-	testCases := []struct {
-		t              reflect.Type
-		expectedResult bool
-	}{
-		{case1Type, true},
-		{case2Type, true},
-		{case3Type, true},
-		{case4Type, true},
-		// Type.Name() starts with lower case and Type.PkgPath() is not empty.
-		{case5Type, false},
-		// Type.Name() starts with lower case and Type.PkgPath() is not empty.
-		{case6Type, false},
-	}
-
-	for i, testCase := range testCases {
-		result := isExportedOrBuiltinType(testCase.t)
-
-		if result != testCase.expectedResult {
-			t.Fatalf("case %v: expected: %v, got: %v\n", i+1, testCase.expectedResult, result)
-		}
-	}
-}
-
-func TestGetMethodMap(t *testing.T) {
-	var a Arith
-	case1Type := reflect.TypeOf(a)
-
-	var aptr *Arith
-	case2Type := reflect.TypeOf(aptr)
-
-	var m mytype
-	case3Type := reflect.TypeOf(m)
-
-	var mptr *mytype
-	case4Type := reflect.TypeOf(mptr)
-
-	testCases := []struct {
-		t              reflect.Type
-		expectedResult int
-	}{
-		// No methods exported.
-		{case1Type, 0},
-		// Multiply and Divide methods are exported.
-		{case2Type, 2},
-		// Foo method is exported.
-		{case3Type, 1},
-		// Foo method is exported.
-		{case4Type, 1},
-	}
-
-	for i, testCase := range testCases {
-		m := getMethodMap(testCase.t)
-		result := len(m)
-
-		if result != testCase.expectedResult {
-			t.Fatalf("case %v: expected: %v, got: %v\n", i+1, testCase.expectedResult, result)
-		}
-	}
-}
-
-func TestServerRegisterName(t *testing.T) {
-	case1Receiver := &Arith{}
-	var case2Receiver mytype
-	var case3Receiver *Arith
-	i := 0
-	var case4Receiver = &i
-	var case5Receiver Arith
-
-	testCases := []struct {
-		name      string
-		receiver  interface{}
-		expectErr bool
-	}{
-		{"Arith", case1Receiver, false},
-		{"arith", case1Receiver, false},
-		{"Arith", case2Receiver, false},
-		// nil receiver error.
-		{"Arith", nil, true},
-		// nil receiver error.
-		{"Arith", case3Receiver, true},
-		// rpc.Register: type Arith has no exported methods of suitable type error.
-		{"Arith", case4Receiver, true},
-		// rpc.Register: type Arith has no exported methods of suitable type (hint: pass a pointer to value of that type) error.
-		{"Arith", case5Receiver, true},
-	}
-
-	for i, testCase := range testCases {
-		err := NewServer().RegisterName(testCase.name, testCase.receiver)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v\n", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func TestServerCall(t *testing.T) {
-	server1 := NewServer()
-	if err := server1.RegisterName("Arith", &Arith{}); err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	server2 := NewServer()
-	if err := server2.RegisterName("arith", &Arith{}); err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	case1ArgBytes, err := gobEncode(&Args{7, 8})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	reply := 7 * 8
-	case1ExpectedResult, err := gobEncode(&reply)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	case2ArgBytes, err := gobEncode(&Args{})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	testCases := []struct {
-		server         *Server
-		serviceMethod  string
-		argBytes       []byte
-		expectedResult []byte
-		expectErr      bool
-	}{
-		{server1, "Arith.Multiply", case1ArgBytes, case1ExpectedResult, false},
-		{server2, "arith.Multiply", case1ArgBytes, case1ExpectedResult, false},
-		// invalid service/method request ill-formed error.
-		{server1, "Multiply", nil, nil, true},
-		// can't find service error.
-		{server1, "arith.Multiply", nil, nil, true},
-		// can't find method error.
-		{server1, "Arith.Add", nil, nil, true},
-		// gob decode error.
-		{server1, "Arith.Multiply", []byte{10}, nil, true},
-		// authentication error.
-		{server1, "Arith.Multiply", case2ArgBytes, nil, true},
-	}
-
-	for i, testCase := range testCases {
-		buf := bytes.NewBuffer([]byte{})
-
-		err := testCase.server.call(testCase.serviceMethod, testCase.argBytes, buf)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: error: expected: %v, got: %v\n", i+1, testCase.expectErr, expectErr)
-		}
-
-		if !testCase.expectErr {
-			if !reflect.DeepEqual(buf.Bytes(), testCase.expectedResult) {
-				t.Fatalf("case %v: result: expected: %v, got: %v\n", i+1, testCase.expectedResult, buf.Bytes())
-			}
-		}
-	}
-}
-
-func TestServerServeHTTP(t *testing.T) {
-	server1 := NewServer()
-	if err := server1.RegisterName("Arith", &Arith{}); err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	argBytes, err := gobEncode(&Args{7, 8})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	requestBodyData, err := gobEncode(CallRequest{Method: "Arith.Multiply", ArgBytes: argBytes})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	case1Request, err := http.NewRequest("POST", "http://localhost:12345/", bytes.NewReader(requestBodyData))
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	reply := 7 * 8
-	replyBytes, err := gobEncode(&reply)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	case1Result, err := gobEncode(CallResponse{ReplyBytes: replyBytes})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	case2Request, err := http.NewRequest("GET", "http://localhost:12345/", bytes.NewReader([]byte{}))
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	case3Request, err := http.NewRequest("POST", "http://localhost:12345/", bytes.NewReader([]byte{10, 20}))
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	requestBodyData, err = gobEncode(CallRequest{Method: "Arith.Add", ArgBytes: argBytes})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	case4Request, err := http.NewRequest("POST", "http://localhost:12345/", bytes.NewReader(requestBodyData))
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	case4Result, err := gobEncode(CallResponse{Error: "can't find method Add"})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	testCases := []struct {
-		server         *Server
-		httpRequest    *http.Request
-		expectedCode   int
-		expectedResult []byte
-	}{
-		{server1, case1Request, http.StatusOK, case1Result},
-		{server1, case2Request, http.StatusMethodNotAllowed, nil},
-		{server1, case3Request, http.StatusBadRequest, nil},
-		{server1, case4Request, http.StatusOK, case4Result},
-	}
-
-	for i, testCase := range testCases {
-		writer := httptest.NewRecorder()
-		testCase.server.ServeHTTP(writer, testCase.httpRequest)
-		if writer.Code != testCase.expectedCode {
-			t.Fatalf("case %v: code: expected: %v, got: %v\n", i+1, testCase.expectedCode, writer.Code)
-		}
-
-		if testCase.expectedCode == http.StatusOK {
-			result := writer.Body.Bytes()
-			if !reflect.DeepEqual(result, testCase.expectedResult) {
-				t.Fatalf("case %v: result: expected: %v, got: %v\n", i+1, testCase.expectedResult, result)
-			}
-		}
-	}
-}

cmd/rpc_test.go

@@ -1,402 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/tls"
-	"errors"
-	"net/http"
-	"net/http/httptest"
-	"reflect"
-	"testing"
-	"time"
-
-	xrpc "github.com/minio/minio/cmd/rpc"
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-func TestAuthArgsAuthenticate(t *testing.T) {
-	tmpGlobalServerConfig := globalServerConfig
-	defer func() {
-		globalServerConfig = tmpGlobalServerConfig
-	}()
-	globalServerConfig = newServerConfig()
-
-	case1Args := AuthArgs{
-		Token:       newAuthToken(),
-		RPCVersion:  globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-
-	case2Args := AuthArgs{
-		Token:       newAuthToken(),
-		RPCVersion:  globalRPCAPIVersion,
-		RequestTime: UTCNow().Add(15 * time.Minute),
-	}
-
-	case3Args := AuthArgs{
-		Token:       newAuthToken(),
-		RPCVersion:  globalRPCAPIVersion,
-		RequestTime: UTCNow().Add(-16 * time.Minute),
-	}
-
-	case4Args := AuthArgs{
-		Token:       newAuthToken(),
-		RPCVersion:  RPCVersion{99, 99, 99},
-		RequestTime: UTCNow(),
-	}
-
-	case5Args := AuthArgs{
-		Token:       "invalid-token",
-		RPCVersion:  globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-
-	testCases := []struct {
-		args      AuthArgs
-		expectErr bool
-	}{
-		{case1Args, false},
-		{case2Args, false},
-		{case3Args, true},
-		{case4Args, true},
-		{case5Args, true},
-	}
-
-	for i, testCase := range testCases {
-		err := testCase.args.Authenticate()
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func TestAuthArgsSetAuthArgs(t *testing.T) {
-	tmpGlobalServerConfig := globalServerConfig
-	defer func() {
-		globalServerConfig = tmpGlobalServerConfig
-	}()
-	globalServerConfig = newServerConfig()
-
-	case1Args := AuthArgs{
-		Token:       newAuthToken(),
-		RPCVersion:  globalRPCAPIVersion,
-		RequestTime: UTCNow(),
-	}
-
-	case2Args := AuthArgs{
-		Token:       newAuthToken(),
-		RPCVersion:  globalRPCAPIVersion,
-		RequestTime: UTCNow().Add(15 * time.Minute),
-	}
-
-	testCases := []struct {
-		args           *AuthArgs
-		authArgs       AuthArgs
-		expectedResult *AuthArgs
-	}{
-		{&AuthArgs{}, case1Args, &case1Args},
-		{&case2Args, case1Args, &case1Args},
-	}
-
-	for i, testCase := range testCases {
-		testCase.args.SetAuthArgs(testCase.authArgs)
-		result := testCase.args
-
-		if !reflect.DeepEqual(result, testCase.expectedResult) {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectedResult, result)
-		}
-	}
-}
-
-func TestRPCClientArgsValidate(t *testing.T) {
-	case1URL, err := xnet.ParseURL("http://localhost:12345/rpc")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	case1Args := RPCClientArgs{
-		NewAuthTokenFunc: newAuthToken,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceName:      "Arith",
-		ServiceURL:       case1URL,
-		TLSConfig:        nil,
-	}
-
-	case2URL, err := xnet.ParseURL("https://localhost:12345/rpc")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	case2Args := RPCClientArgs{
-		NewAuthTokenFunc: newAuthToken,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceName:      "Arith",
-		ServiceURL:       case1URL,
-		TLSConfig:        &tls.Config{},
-	}
-
-	case3Args := RPCClientArgs{
-		NewAuthTokenFunc: nil,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceName:      "Arith",
-		ServiceURL:       case1URL,
-		TLSConfig:        &tls.Config{},
-	}
-
-	case4Args := RPCClientArgs{
-		NewAuthTokenFunc: newAuthToken,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceURL:       case1URL,
-		TLSConfig:        &tls.Config{},
-	}
-
-	case5URL, err := xnet.ParseURL("ftp://localhost:12345/rpc")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	case5Args := RPCClientArgs{
-		NewAuthTokenFunc: newAuthToken,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceName:      "Arith",
-		ServiceURL:       case5URL,
-		TLSConfig:        &tls.Config{},
-	}
-
-	case6URL, err := xnet.ParseURL("http://localhost:12345/rpc?location")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	case6Args := RPCClientArgs{
-		NewAuthTokenFunc: newAuthToken,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceName:      "Arith",
-		ServiceURL:       case6URL,
-		TLSConfig:        &tls.Config{},
-	}
-
-	case7Args := RPCClientArgs{
-		NewAuthTokenFunc: newAuthToken,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceName:      "Arith",
-		ServiceURL:       case2URL,
-		TLSConfig:        nil,
-	}
-
-	testCases := []struct {
-		args      RPCClientArgs
-		expectErr bool
-	}{
-		{case1Args, false},
-		{case2Args, false},
-		// NewAuthTokenFunc must not be empty error.
-		{case3Args, true},
-		// ServiceName must not be empty.
-		{case4Args, true},
-		// unknown RPC URL error.
-		{case5Args, true},
-		// unknown RPC URL error.
-		{case6Args, true},
-		// tls configuration must not be empty for https url error.
-		{case7Args, true},
-	}
-
-	for i, testCase := range testCases {
-		err := testCase.args.validate()
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-type Args struct {
-	AuthArgs
-	A, B int
-}
-
-type Quotient struct {
-	Quo, Rem int
-}
-
-type Arith struct{}
-
-func (t *Arith) Multiply(args *Args, reply *int) error {
-	*reply = args.A * args.B
-	return nil
-}
-
-func (t *Arith) Divide(args *Args, quo *Quotient) error {
-	if args.B == 0 {
-		return errors.New("divide by zero")
-	}
-	quo.Quo = args.A / args.B
-	quo.Rem = args.A % args.B
-	return nil
-}
-
-func TestRPCClientCall(t *testing.T) {
-	tmpGlobalServerConfig := globalServerConfig
-	defer func() {
-		globalServerConfig = tmpGlobalServerConfig
-	}()
-	globalServerConfig = newServerConfig()
-
-	rpcServer := xrpc.NewServer()
-	if err := rpcServer.RegisterName("Arith", &Arith{}); err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		rpcServer.ServeHTTP(w, r)
-	}))
-	defer httpServer.Close()
-
-	url, err := xnet.ParseURL(httpServer.URL)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	rpcClient, err := NewRPCClient(RPCClientArgs{
-		NewAuthTokenFunc: newAuthToken,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceName:      "Arith",
-		ServiceURL:       url,
-	})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	var case1Result int
-	case1ExpectedResult := 19 * 8
-
-	testCases := []struct {
-		serviceMethod  string
-		args           *Args
-		result         interface{}
-		changeConfig   bool
-		expectedResult interface{}
-		expectErr      bool
-	}{
-		{"Arith.Multiply", &Args{A: 19, B: 8}, &case1Result, false, &case1ExpectedResult, false},
-		{"Arith.Divide", &Args{A: 19, B: 8}, &Quotient{}, false, &Quotient{2, 3}, false},
-		{"Arith.Multiply", &Args{A: 19, B: 8}, &case1Result, true, &case1ExpectedResult, false},
-		{"Arith.Divide", &Args{A: 19, B: 8}, &Quotient{}, true, &Quotient{2, 3}, false},
-		{"Arith.Divide", &Args{A: 19, B: 0}, &Quotient{}, false, nil, true},
-		{"Arith.Divide", &Args{A: 19, B: 8}, &case1Result, false, nil, true},
-	}
-
-	for i, testCase := range testCases {
-		if testCase.changeConfig {
-			globalServerConfig = newServerConfig()
-		}
-
-		err := rpcClient.Call(testCase.serviceMethod, testCase.args, testCase.result)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-
-		if !testCase.expectErr {
-			if !reflect.DeepEqual(testCase.result, testCase.expectedResult) {
-				t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectedResult, testCase.result)
-			}
-		}
-	}
-}
-
-func TestRPCClientClose(t *testing.T) {
-	tmpGlobalServerConfig := globalServerConfig
-	defer func() {
-		globalServerConfig = tmpGlobalServerConfig
-	}()
-	globalServerConfig = newServerConfig()
-
-	url, err := xnet.ParseURL("http://localhost:12345")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	rpcClient, err := NewRPCClient(RPCClientArgs{
-		NewAuthTokenFunc: newAuthToken,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceName:      "Arith",
-		ServiceURL:       url,
-	})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	testCases := []struct {
-		rpcClient *RPCClient
-		expectErr bool
-	}{
-		{rpcClient, false},
-		// Double close.
-		{rpcClient, false},
-	}
-
-	for i, testCase := range testCases {
-		err := testCase.rpcClient.Close()
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func TestRPCClientServiceURL(t *testing.T) {
-	tmpGlobalServerConfig := globalServerConfig
-	defer func() {
-		globalServerConfig = tmpGlobalServerConfig
-	}()
-	globalServerConfig = newServerConfig()
-
-	url, err := xnet.ParseURL("http://localhost:12345")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	rpcClient, err := NewRPCClient(RPCClientArgs{
-		NewAuthTokenFunc: newAuthToken,
-		RPCVersion:       globalRPCAPIVersion,
-		ServiceName:      "Arith",
-		ServiceURL:       url,
-	})
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	case1Result, err := xnet.ParseURL("http://localhost:12345")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	testCases := []struct {
-		rpcClient      *RPCClient
-		expectedResult *xnet.URL
-	}{
-		{rpcClient, case1Result},
-	}
-
-	for i, testCase := range testCases {
-		result := testCase.rpcClient.ServiceURL()
-
-		if !reflect.DeepEqual(result, testCase.expectedResult) {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectedResult, result)
-		}
-	}
-}

cmd/server-main.go

@@ -101,36 +101,36 @@ ENVIRONMENT VARIABLES:
 
 EXAMPLES:
   1. Start minio server on "/home/shared" directory.
-     $ {{.HelpName}} /home/shared
+     {{.Prompt}} {{.HelpName}} /home/shared
 
   2. Start minio server bound to a specific ADDRESS:PORT.
-     $ {{.HelpName}} --address 192.168.1.101:9000 /home/shared
+     {{.Prompt}} {{.HelpName}} --address 192.168.1.101:9000 /home/shared
 
   3. Start minio server and enable virtual-host-style requests.
-     $ export MINIO_DOMAIN=mydomain.com
-     $ {{.HelpName}} --address mydomain.com:9000 /mnt/export
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_DOMAIN{{.AssignmentOperator}}mydomain.com
+     {{.Prompt}} {{.HelpName}} --address mydomain.com:9000 /mnt/export
 
   4. Start erasure coded minio server on a node with 64 drives.
-     $ {{.HelpName}} /mnt/export{1...64}
+     {{.Prompt}} {{.HelpName}} /mnt/export{1...64}
 
   5. Start distributed minio server on an 32 node setup with 32 drives each. Run following command on all the 32 nodes.
-     $ export MINIO_ACCESS_KEY=minio
-     $ export MINIO_SECRET_KEY=miniostorage
-     $ {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32}
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}minio
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}miniostorage
+     {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32}
 
   6. Start minio server with edge caching enabled.
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
-     $ export MINIO_CACHE_EXPIRY=40
-     $ export MINIO_CACHE_MAXUSE=80
-     $ {{.HelpName}} /home/shared
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
+     {{.Prompt}} {{.HelpName}} /home/shared
 
   7. Start minio server with KMS enabled.
-     $ export MINIO_SSE_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
-     $ export MINIO_SSE_VAULT_APPROLE_SECRET=4e30c52f-13e4-a6f5-0763-d50e8cb4321f
-     $ export MINIO_SSE_VAULT_ENDPOINT=https://vault-endpoint-ip:8200
-     $ export MINIO_SSE_VAULT_KEY_NAME=my-minio-key
-     $ {{.HelpName}} /home/shared
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_ID{{.AssignmentOperator}}9b56cc08-8258-45d5-24a3-679876769126
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_SECRET{{.AssignmentOperator}}4e30c52f-13e4-a6f5-0763-d50e8cb4321f
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_ENDPOINT{{.AssignmentOperator}}https://vault-endpoint-ip:8200
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_KEY_NAME{{.AssignmentOperator}}my-minio-key
+     {{.Prompt}} {{.HelpName}} /home/shared
 `,
 }
 
@@ -167,6 +167,8 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	}
 	logger.FatalIf(err, "Invalid command line arguments")
 
+	logger.LogIf(context.Background(), checkEndpointsSubOptimal(ctx, setupType, globalEndpoints))
+
 	globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
 
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
@@ -200,6 +202,8 @@ func serverMain(ctx *cli.Context) {
 		cli.ShowCommandHelpAndExit(ctx, "server", 1)
 	}
 
+	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
+
 	// Disable logging until server initialization is complete, any
 	// error during initialization will be shown as a fatal message
 	logger.Disable = true
@@ -305,8 +309,6 @@ func serverMain(ctx *cli.Context) {
 		globalHTTPServerErrorCh <- globalHTTPServer.Start()
 	}()
 
-	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
-
 	newObject, err := newObjectLayer(globalEndpoints)
 	if err != nil {
 		// Stop watching for any certificate changes.

cmd/server_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/xml"
@@ -2725,12 +2724,9 @@ func (s *TestSuiteCommon) TestObjectMultipart(c *check) {
 	c.Assert(response.StatusCode, http.StatusOK)
 	var parts []CompletePart
 	for _, part := range completeUploads.Parts {
-		// For compressed objects, we dont treat E-Tag as checksum.
-		part.ETag = strings.Replace(part.ETag, "-1", "", -1)
 		part.ETag = canonicalizeETag(part.ETag)
 		parts = append(parts, part)
 	}
-	etag, err := getCompleteMultipartMD5(context.Background(), parts)
-	c.Assert(err, nil)
+	etag := getCompleteMultipartMD5(parts)
 	c.Assert(canonicalizeETag(response.Header.Get("Etag")), etag)
 }

cmd/signature-v4-utils.go

@@ -146,6 +146,7 @@ func sumHMAC(key []byte, data []byte) []byte {
 // extractSignedHeaders extract signed headers from Authorization header
 func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header, APIErrorCode) {
 	reqHeaders := r.Header
+	reqQueries := r.URL.Query()
 	// find whether "host" is part of list of signed headers.
 	// if not return ErrUnsignedHeaders. "host" is mandatory.
 	if !contains(signedHeaders, "host") {
@@ -156,6 +157,10 @@ func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header,
 		// `host` will not be found in the headers, can be found in r.Host.
 		// but its alway necessary that the list of signed headers containing host in it.
 		val, ok := reqHeaders[http.CanonicalHeaderKey(header)]
+		if !ok {
+			// try to set headers from Query String
+			val, ok = reqQueries[header]
+		}
 		if ok {
 			for _, enc := range val {
 				extractedSignedHeaders.Add(header, enc)

cmd/signature-v4-utils_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"github.com/minio/minio/cmd/crypto"
 	"net/http"
 	"testing"
 )
@@ -149,6 +150,22 @@ func TestExtractSignedHeaders(t *testing.T) {
 		t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode)
 	}
 
+	inputQuery := r.URL.Query()
+	// case where some headers need to get from request query
+	signedHeaders = append(signedHeaders, "x-amz-server-side-encryption")
+	// expect to fail with `ErrUnsignedHeaders` because couldn't find some header
+	_, errCode = extractSignedHeaders(signedHeaders, r)
+	if errCode != ErrUnsignedHeaders {
+		t.Fatalf("Expected the APIErrorCode to %d, but got %d", ErrUnsignedHeaders, errCode)
+	}
+	// set headers value through Get parameter
+	inputQuery.Add("x-amz-server-side-encryption", crypto.SSEAlgorithmAES256)
+	r.URL.RawQuery = inputQuery.Encode()
+	_, errCode = extractSignedHeaders(signedHeaders, r)
+	if errCode != ErrNone {
+		t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode)
+	}
+
 	// "x-amz-content-sha256" header value from the extracted result.
 	extractedContentSha256 := extractedSignedHeaders.Get("x-amz-content-sha256")
 	// "host" header value from the extracted result.

cmd/signature-v4.go

@@ -251,13 +251,19 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 
 	// Save other headers available in the request parameters.
 	for k, v := range req.URL.Query() {
+		key := strings.ToLower(k)
 
 		// Handle the metadata in presigned put query string
-		if strings.Contains(strings.ToLower(k), "x-amz-meta-") {
+		if strings.Contains(key, "x-amz-meta-") {
+			query.Set(k, v[0])
+			continue
+		}
+
+		if strings.Contains(key, "x-amz-server-side-") {
 			query.Set(k, v[0])
 		}
 
-		if strings.HasPrefix(strings.ToLower(k), "x-amz") {
+		if strings.HasPrefix(key, "x-amz") {
 			continue
 		}
 		query[k] = v

cmd/storage-datatypes.go

@@ -30,6 +30,13 @@ type VolInfo struct {
 	Created time.Time
 }
 
+// FilesInfo represent a list of files, additionally
+// indicates if the list is last.
+type FilesInfo struct {
+	Files       []FileInfo
+	IsTruncated bool
+}
+
 // FileInfo - represents file stat information.
 type FileInfo struct {
 	// Name of the volume.
@@ -46,4 +53,12 @@ type FileInfo struct {
 
 	// File mode bits.
 	Mode os.FileMode
+
+	// File metadata
+	Metadata map[string]string
+
+	// All the parts per object.
+	Parts []ObjectPartInfo
+
+	Quorum int
 }

cmd/storage-errors.go

@@ -48,6 +48,9 @@ var errDiskAccessDenied = errors.New("disk access denied")
 // errFileNotFound - cannot find the file.
 var errFileNotFound = errors.New("file not found")
 
+// errTooManyOpenFiles - too many open files.
+var errTooManyOpenFiles = errors.New("too many open files")
+
 // errFileNameTooLong - given file name is too long than supported length.
 var errFileNameTooLong = errors.New("file name too long")
 

cmd/storage-interface.go

@@ -38,8 +38,12 @@ type StorageAPI interface {
 	StatVol(volume string) (vol VolInfo, err error)
 	DeleteVol(volume string) (err error)
 
+	// Walk in sorted order directly on disk.
+	Walk(volume, dirPath string, marker string, recursive bool, leafFile string,
+		readMetadataFn readMetadataFunc, endWalkCh chan struct{}) (chan FileInfo, error)
+
 	// File operations.
-	ListDir(volume, dirPath string, count int) ([]string, error)
+	ListDir(volume, dirPath string, count int, leafFile string) ([]string, error)
 	ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error)
 	AppendFile(volume string, path string, buf []byte) (err error)
 	CreateFile(volume, path string, size int64, reader io.Reader) error
@@ -47,9 +51,10 @@ type StorageAPI interface {
 	RenameFile(srcVolume, srcPath, dstVolume, dstPath string) error
 	StatFile(volume string, path string) (file FileInfo, err error)
 	DeleteFile(volume string, path string) (err error)
+	DeleteFileBulk(volume string, paths []string) (errs []error, err error)
 
 	// Write all data, syncs the data to disk.
-	WriteAll(volume string, path string, buf []byte) (err error)
+	WriteAll(volume string, path string, reader io.Reader) (err error)
 
 	// Read all.
 	ReadAll(volume string, path string) (buf []byte, err error)

cmd/storage-rest-client.go

@@ -44,6 +44,9 @@ func isNetworkError(err error) bool {
 	if err.Error() == errConnectionStale.Error() {
 		return true
 	}
+	if strings.Contains(err.Error(), "connection reset by peer") {
+		return true
+	}
 	if uerr, isURLError := err.(*url.Error); isURLError {
 		if uerr.Timeout() {
 			return true
@@ -56,6 +59,19 @@ func isNetworkError(err error) bool {
 	return isNetOpError
 }
 
+// Attempt to approximate network error with a
+// typed network error, otherwise default to
+// errDiskNotFound
+func toNetworkError(err error) error {
+	if err == nil {
+		return err
+	}
+	if strings.Contains(err.Error(), "connection reset by peer") {
+		return errNetworkConnReset
+	}
+	return errDiskNotFound
+}
+
 // Converts rpc.ServerError to underlying error. This function is
 // written so that the storageAPI errors are consistent across network
 // disks as well.
@@ -65,7 +81,7 @@ func toStorageErr(err error) error {
 	}
 
 	if isNetworkError(err) {
-		return errDiskNotFound
+		return toNetworkError(err)
 	}
 
 	switch err.Error() {
@@ -234,17 +250,16 @@ func (client *storageRESTClient) CreateFile(volume, path string, length int64, r
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
 	values.Set(storageRESTLength, strconv.Itoa(int(length)))
-	respBody, err := client.call(storageRESTMethodCreateFile, values, r, length)
+	respBody, err := client.call(storageRESTMethodCreateFile, values, ioutil.NopCloser(r), length)
 	defer http.DrainBody(respBody)
 	return err
 }
 
 // WriteAll - write all data to a file.
-func (client *storageRESTClient) WriteAll(volume, path string, buffer []byte) error {
+func (client *storageRESTClient) WriteAll(volume, path string, reader io.Reader) error {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
-	reader := bytes.NewBuffer(buffer)
 	respBody, err := client.call(storageRESTMethodWriteAll, values, reader, -1)
 	defer http.DrainBody(respBody)
 	return err
@@ -314,12 +329,50 @@ func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buf
 	return int64(n), err
 }
 
+func (client *storageRESTClient) Walk(volume, dirPath, marker string, recursive bool, leafFile string,
+	readMetadataFn readMetadataFunc, endWalkCh chan struct{}) (chan FileInfo, error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTDirPath, dirPath)
+	values.Set(storageRESTMarkerPath, marker)
+	values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
+	values.Set(storageRESTLeafFile, leafFile)
+	respBody, err := client.call(storageRESTMethodWalk, values, nil, -1)
+	if err != nil {
+		return nil, err
+	}
+
+	ch := make(chan FileInfo)
+	go func() {
+		defer close(ch)
+		defer http.DrainBody(respBody)
+
+		decoder := gob.NewDecoder(respBody)
+		for {
+			var fi FileInfo
+			if gerr := decoder.Decode(&fi); gerr != nil {
+				// Upon error return
+				return
+			}
+			select {
+			case ch <- fi:
+			case <-endWalkCh:
+				return
+			}
+
+		}
+	}()
+
+	return ch, nil
+}
+
 // ListDir - lists a directory.
-func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (entries []string, err error) {
+func (client *storageRESTClient) ListDir(volume, dirPath string, count int, leafFile string) (entries []string, err error) {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTDirPath, dirPath)
 	values.Set(storageRESTCount, strconv.Itoa(count))
+	values.Set(storageRESTLeafFile, leafFile)
 	respBody, err := client.call(storageRESTMethodListDir, values, nil, -1)
 	if err != nil {
 		return nil, err
@@ -339,6 +392,34 @@ func (client *storageRESTClient) DeleteFile(volume, path string) error {
 	return err
 }
 
+// DeleteFileBulk - deletes files in bulk.
+func (client *storageRESTClient) DeleteFileBulk(volume string, paths []string) (errs []error, err error) {
+	errs = make([]error, len(paths))
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	for _, path := range paths {
+		values.Add(storageRESTFilePath, path)
+	}
+	respBody, err := client.call(storageRESTMethodDeleteFileBulk, values, nil, -1)
+	defer http.DrainBody(respBody)
+
+	if err != nil {
+		return nil, err
+	}
+
+	bulkErrs := bulkErrorsResponse{}
+	gob.NewDecoder(respBody).Decode(&bulkErrs)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, dErr := range bulkErrs.Errs {
+		errs[i] = toStorageErr(dErr)
+	}
+
+	return errs, nil
+}
+
 // RenameFile - renames a file.
 func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
 	values := make(url.Values)
@@ -405,6 +486,7 @@ func newStorageRESTClient(endpoint Endpoint) (*storageRESTClient, error) {
 		tlsConfig = &tls.Config{
 			ServerName: host.Name,
 			RootCAs:    globalRootCAs,
+			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 

cmd/storage-rest-common.go

@@ -16,7 +16,7 @@
 
 package cmd
 
-const storageRESTVersion = "v4"
+const storageRESTVersion = "v6"
 const storageRESTPath = minioReservedBucketPath + "/storage/" + storageRESTVersion + "/"
 
 const (
@@ -34,7 +34,9 @@ const (
 	storageRESTMethodReadFile       = "readfile"
 	storageRESTMethodReadFileStream = "readfilestream"
 	storageRESTMethodListDir        = "listdir"
+	storageRESTMethodWalk           = "walk"
 	storageRESTMethodDeleteFile     = "deletefile"
+	storageRESTMethodDeleteFileBulk = "deletefilebulk"
 	storageRESTMethodRenameFile     = "renamefile"
 	storageRESTMethodGetInstanceID  = "getinstanceid"
 )
@@ -50,6 +52,9 @@ const (
 	storageRESTOffset     = "offset"
 	storageRESTLength     = "length"
 	storageRESTCount      = "count"
+	storageRESTMarkerPath = "marker"
+	storageRESTLeafFile   = "leaf-file"
+	storageRESTRecursive  = "recursive"
 	storageRESTBitrotAlgo = "bitrot-algo"
 	storageRESTBitrotHash = "bitrot-hash"
 	storageRESTInstanceID = "instance-id"

cmd/storage-rest-server.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"encoding/gob"
 	"encoding/hex"
 	"errors"
@@ -47,6 +48,25 @@ func (s *storageRESTServer) writeErrorResponse(w http.ResponseWriter, err error)
 	w.(http.Flusher).Flush()
 }
 
+type bulkErrorsResponse struct {
+	Errs []error `json:"errors"`
+}
+
+func (s *storageRESTServer) writeErrorsResponse(w http.ResponseWriter, errs []error) {
+	resp := bulkErrorsResponse{Errs: make([]error, len(errs))}
+	for idx, err := range errs {
+		if err == nil {
+			continue
+		}
+		resp.Errs[idx] = err
+	}
+	gob.NewEncoder(w).Encode(resp)
+	w.(http.Flusher).Flush()
+}
+
+// DefaultSkewTime - skew time is 15 minutes between minio peers.
+const DefaultSkewTime = 15 * time.Minute
+
 // Authenticates storage client's requests and validates for skewed time.
 func storageServerRequestValidate(r *http.Request) error {
 	_, owner, err := webRequestAuthenticate(r)
@@ -225,14 +245,7 @@ func (s *storageRESTServer) WriteAllHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	buf := make([]byte, r.ContentLength)
-	_, err := io.ReadFull(r.Body, buf)
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	err = s.storage.WriteAll(volume, filePath, buf)
+	err := s.storage.WriteAll(volume, filePath, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		s.writeErrorResponse(w, err)
 	}
@@ -350,6 +363,57 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
 	w.(http.Flusher).Flush()
 }
 
+// readMetadata func provides the function types for reading leaf metadata.
+type readMetadataFunc func(buf []byte, volume, entry string) FileInfo
+
+func readMetadata(buf []byte, volume, entry string) FileInfo {
+	m, err := xlMetaV1UnmarshalJSON(context.Background(), buf)
+	if err != nil {
+		return FileInfo{}
+	}
+	return FileInfo{
+		Volume:   volume,
+		Name:     entry,
+		ModTime:  m.Stat.ModTime,
+		Size:     m.Stat.Size,
+		Metadata: m.Meta,
+		Parts:    m.Parts,
+		Quorum:   m.Erasure.DataBlocks,
+	}
+}
+
+// WalkHandler - remote caller to start walking at a requested directory path.
+func (s *storageRESTServer) WalkHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	dirPath := vars[storageRESTDirPath]
+	markerPath := vars[storageRESTMarkerPath]
+	recursive, err := strconv.ParseBool(vars[storageRESTRecursive])
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	leafFile := vars[storageRESTLeafFile]
+
+	endWalkCh := make(chan struct{})
+	defer close(endWalkCh)
+
+	fch, err := s.storage.Walk(volume, dirPath, markerPath, recursive, leafFile, readMetadata, endWalkCh)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	defer w.(http.Flusher).Flush()
+
+	encoder := gob.NewEncoder(w)
+	for fi := range fch {
+		encoder.Encode(&fi)
+	}
+}
+
 // ListDirHandler - list a directory.
 func (s *storageRESTServer) ListDirHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -358,12 +422,13 @@ func (s *storageRESTServer) ListDirHandler(w http.ResponseWriter, r *http.Reques
 	vars := mux.Vars(r)
 	volume := vars[storageRESTVolume]
 	dirPath := vars[storageRESTDirPath]
+	leafFile := vars[storageRESTLeafFile]
 	count, err := strconv.Atoi(vars[storageRESTCount])
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-	entries, err := s.storage.ListDir(volume, dirPath, count)
+	entries, err := s.storage.ListDir(volume, dirPath, count, leafFile)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
@@ -387,6 +452,24 @@ func (s *storageRESTServer) DeleteFileHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
+// DeleteFileBulkHandler - delete a file.
+func (s *storageRESTServer) DeleteFileBulkHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := r.URL.Query()
+	volume := vars.Get(storageRESTVolume)
+	filePaths := vars[storageRESTFilePath]
+
+	errs, err := s.storage.DeleteFileBulk(volume, filePaths)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	s.writeErrorsResponse(w, errs)
+}
+
 // RenameFileHandler - rename a file.
 func (s *storageRESTServer) RenameFileHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -440,9 +523,14 @@ func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodReadFileStream).HandlerFunc(httpTraceHdrs(server.ReadFileStreamHandler)).
 			Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTOffset, storageRESTLength)...)
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodListDir).HandlerFunc(httpTraceHdrs(server.ListDirHandler)).
-			Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTCount)...)
+			Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTCount, storageRESTLeafFile)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodWalk).HandlerFunc(httpTraceHdrs(server.WalkHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTMarkerPath, storageRESTRecursive, storageRESTLeafFile)...)
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodDeleteFile).HandlerFunc(httpTraceHdrs(server.DeleteFileHandler)).
 			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodDeleteFileBulk).HandlerFunc(httpTraceHdrs(server.DeleteFileBulkHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
+
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
 			Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodGetInstanceID).HandlerFunc(httpTraceAll(server.GetInstanceID))

cmd/storage-rest_test.go

@@ -260,7 +260,7 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) {
 	}
 
 	for i, testCase := range testCases {
-		result, err := storage.ListDir(testCase.volumeName, testCase.prefix, -1)
+		result, err := storage.ListDir(testCase.volumeName, testCase.prefix, -1, "")
 		expectErr := (err != nil)
 
 		if expectErr != testCase.expectErr {
@@ -508,6 +508,10 @@ func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRES
 		t.Fatalf("NewEndpoint failed %v", endpoint)
 	}
 
+	if err := endpoint.UpdateIsLocal(); err != nil {
+		t.Fatalf("UpdateIsLocal failed %v", err)
+	}
+
 	registerStorageRESTHandlers(router, EndpointList{endpoint})
 	restClient, err := newStorageRESTClient(endpoint)
 	if err != nil {

cmd/sts-handlers.go

@@ -25,6 +25,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/iam/validator"
+	"github.com/minio/minio/pkg/wildcard"
 )
 
 const (
@@ -49,13 +50,19 @@ func registerSTSRouter(router *mux.Router) {
 	stsRouter := router.NewRoute().PathPrefix("/").Subrouter()
 
 	// Assume roles with no JWT, handles AssumeRole.
-	stsRouter.Methods("POST").HeadersRegexp("Content-Type", "application/x-www-form-urlencoded*").
-		HeadersRegexp("Authorization", "AWS4-HMAC-SHA256*").
-		HandlerFunc(httpTraceAll(sts.AssumeRole))
+	stsRouter.Methods("POST").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
+		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get("Content-Type"))
+		authOk := wildcard.MatchSimple("AWS4-HMAC-SHA256*", r.Header.Get("Authorization"))
+		noQueries := len(r.URL.Query()) == 0
+		return ctypeOk && authOk && noQueries
+	}).HandlerFunc(httpTraceAll(sts.AssumeRole))
 
 	// Assume roles with JWT handler, handles both ClientGrants and WebIdentity.
-	stsRouter.Methods("POST").HeadersRegexp("Content-Type", "application/x-www-form-urlencoded*").
-		HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))
+	stsRouter.Methods("POST").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
+		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get("Content-Type"))
+		noQueries := len(r.URL.Query()) == 0
+		return ctypeOk && noQueries
+	}).HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))
 
 	// AssumeRoleWithClientGrants
 	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithClientGrants)).

cmd/test-utils_test.go

@@ -138,7 +138,7 @@ func calculateSignedChunkLength(chunkDataSize int64) int64 {
 }
 
 func mustGetPutObjReader(t TestErrHandler, data io.Reader, size int64, md5hex, sha256hex string) *PutObjReader {
-	hr, err := hash.NewReader(data, size, md5hex, sha256hex, size)
+	hr, err := hash.NewReader(data, size, md5hex, sha256hex, size, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -174,7 +174,7 @@ func prepareFS() (ObjectLayer, string, error) {
 	return obj, fsDirs[0], nil
 }
 
-func prepareXL32() (ObjectLayer, []string, error) {
+func prepareXLSets32() (ObjectLayer, []string, error) {
 	fsDirs1, err := getRandomDisks(16)
 	if err != nil {
 		return nil, nil, err
@@ -1704,7 +1704,7 @@ func prepareTestBackend(instanceType string) (ObjectLayer, []string, error) {
 	switch instanceType {
 	// Total number of disks for XL sets backend is set to 32.
 	case XLSetsTestStr:
-		return prepareXL32()
+		return prepareXLSets32()
 	// Total number of disks for XL backend is set to 16.
 	case XLTestStr:
 		return prepareXL16()
@@ -1927,6 +1927,10 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 
 	globalIAMSys = NewIAMSys()
 	globalIAMSys.Init(objLayer)
+
+	globalPolicySys = NewPolicySys()
+	globalPolicySys.Init(objLayer)
+
 	// initialize the server and obtain the credentials and root.
 	// credentials are necessary to sign the HTTP request.
 	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
@@ -1982,7 +1986,7 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
 	// Executing the object layer tests for single node setup.
 	objTest(objLayer, FSTestStr, t)
 
-	objLayer, fsDirs, err := prepareXL16()
+	objLayer, fsDirs, err := prepareXLSets32()
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
 	}